creation of a backend for osm2pgsql specific felx output

Author: etienneJr

README.md

@@ -6,9 +6,12 @@ Underpass-API aim to be a [Overpass-API](https://github.com/drolbr/Overpass-API)
 
 ## Prepare the data & Run the server
 
-Folow the instruction of one of the backends:
-* [Postgres+PostGIS / Osmosis](backends/postgres_osmosis/README.md), Osmosis schema
+### With docker (recommended)
+
+Follow the instruction of one of the backends:
 * [DuckDB+Spatial / QuackOSM](backends/duckdb_quackosm/README.md), Quackosm schema
+* [Postgres+PostGIS / Osmosis](backends/postgres_osmosis/README.md), Osmosis schema
+* [Postgres+PostGIS / Osm2pgsql](backends/postgres_osm2pgsql/README.md), Osm2pgsql schema using a specific `flex output`
 
 ## Query
 

backends/postgres_osm2pgsql/README.md

@@ -0,0 +1,29 @@
+# Postgres/PostGIS, Osm2pgsql schema
+
+Prepare Docker
+```sh
+docker compose --profile '*' build
+```
+
+## Prepare the data
+
+Create you database with osm2pgsql, using the script `geometries-alone.lua` in this folder.
+
+Example command :
+
+```
+osm2pgsql -U user -d database -c -s --flat-nodes DB-flat-nodes --middle-with-nodes -x -O flex -S geometries-alone.lua extract.osm.pbf
+```
+
+Warning : this backend will not work on a existing osm2pgsql database. it is specific for a database created with the lua script above and the `-s (--slim)` option.
+
+If your database was created "outside" docker, you will have to modify `docker-compose.yaml` to:
+  - delete services `osm2pgsql` and `postgress`
+  - in service api : delete reference `depends on: -postgres` and set your `DATABASE_URL: postgres://user:pw@host:5432/database`
+
+## Run the server
+
+Run the HTTP server
+```
+docker compose up
+```

backends/postgres_osm2pgsql/docker-compose.yaml

@@ -0,0 +1,38 @@
+version: "3.3"
+
+services:
+  osm2pgsql:
+    profiles: [tools]
+    build:
+      context: docker/osm2pgsql
+    environment:
+      DATABASE_URL: postgresql://postgres@postgres:5432/postgres
+    volumes:
+      - ../../data:/data
+    depends_on:
+      - postgres
+
+  postgres:
+    image: postgis/postgis:15-3.4
+    shm_size: 1g
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+    volumes:
+      - ./docker/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+      - pgdata:/var/lib/postgresql/data
+    restart: unless-stopped
+
+  api:
+    extends:
+      file: ../../docker-compose-base.yaml
+      service: api
+    environment:
+      BACKEND: PostgresOsm2pgsql
+      DATABASE_URL: postgresql://postgres@postgres:5432/postgres
+    volumes:
+      - .:/srv/app/backends/postgres_osm2pgsql
+    depends_on:
+      - postgres
+
+volumes:
+  pgdata:

backends/postgres_osm2pgsql/docker/osm2pgsql/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:12
+
+RUN apt update -y && apt install -y \
+    osm2pgsql \
+    postgresql-client

backends/postgres_osm2pgsql/docker/postgres/docker-entrypoint-initdb.d/00_drop-tiger.sql

@@ -0,0 +1,2 @@
+DROP SCHEMA IF EXISTS tiger CASCADE;
+DROP EXTENSION IF EXISTS postgis_tiger_geocoder;

backends/postgres_osm2pgsql/docker/postgres/docker-entrypoint-initdb.d/01_create-hstore.sql

@@ -0,0 +1,8 @@
+CREATE EXTENSION IF NOT EXISTS htsore;
+
+-- Same as ->, for code compatibility with json
+CREATE OPERATOR ->> (
+	LEFTARG = hstore,
+	RIGHTARG = text,
+	PROCEDURE = fetchval
+);

backends/postgres_osm2pgsql/docker/postgres/docker-entrypoint-initdb.d/02_create-postgis.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS postgis;

backends/postgres_osm2pgsql/geometries-alone.lua

@@ -0,0 +1,146 @@
+-- This is a very simple Lua config for the Flex output
+-- which only stores the geometries (not even the tags)
+-- for use with Underpass-API to mimics Overpass
+
+local tables = {}
+
+tables.points = osm2pgsql.define_table({
+    name = 'points',
+    ids = { type = 'node', id_column = 'id' },
+    columns = {
+        -- { column = 'tags', type = 'jsonb' },
+        { column = 'geom', type = 'point', projection = 4326, not_null = true  }
+}})
+
+tables.lines = osm2pgsql.define_table({
+    name = 'lines',
+    ids = { type = 'way', id_column = 'id' },
+    columns = {
+        -- { column = 'tags', type = 'jsonb' },
+        { column = 'geom', type = 'multilinestring', projection = 4326, not_null = true }
+}})
+
+tables.areas = osm2pgsql.define_table({
+    name = 'areas',
+    ids = { type = 'area', id_column = 'id' },
+    columns = {
+        -- { column = 'tags', type = 'jsonb' },
+        { column = 'geom', type = 'geometry', projection = 4326, not_null = true },
+        -- In this column we'll put the true area calculated on the spheroid
+        { column = 'area', type = 'real' }
+}})
+
+
+-- Helper function to remove some of the tags we usually are not interested in.
+-- Returns true if there are no tags left.
+
+-- modifié : retourne vrai si aucun tag
+local function clean_tags(tags)
+    -- tags.odbl = nil
+    -- tags.created_by = nil
+    -- tags.source = nil
+    -- tags['source:ref'] = nil
+
+    return next(tags) == nil
+end
+
+-- Helper function that looks at the tags and decides if this is possibly
+-- an area.
+local function has_area_tags(tags)
+    if tags.area == 'yes' or tags.area == 'true' or tags.area == '1' then
+        return true
+    end
+    if tags.area == 'no' or tags.area == 'false' or tags.area == '0'  then
+        return false
+    end
+
+    return tags.aeroway
+        or tags.amenity
+        or tags.building
+        or tags.harbour
+        or tags.historic
+        or tags.landuse
+        or tags.leisure
+        or tags.man_made
+        or tags.military
+        or tags.natural
+        or tags.office
+        or tags.place
+        or tags.power
+        or tags.public_transport
+        or tags.shop
+        or tags.sport
+        or tags.tourism
+        or tags.water
+        or tags.waterway
+        or tags.wetland
+        or tags['abandoned:aeroway']
+        or tags['abandoned:amenity']
+        or tags['abandoned:building']
+        or tags['abandoned:landuse']
+        or tags['abandoned:power']
+        or tags['area:highway']
+end
+
+function osm2pgsql.process_node(object)
+    if clean_tags(object.tags) then
+        return
+    end
+
+    local geom = object:as_point()
+
+    tables.points:insert({
+        -- tags = object.tags,
+        geom = geom -- the point will be automatically be projected to 3857
+    })
+
+end
+
+function osm2pgsql.process_way(object)
+    if clean_tags(object.tags) then
+        return
+    end
+
+    -- A closed way that also has the right tags for an area is a polygon.
+    if object.is_closed and has_area_tags(object.tags) then
+        -- Creating the polygon geometry takes time, so we do it once here
+        -- and later store it in the table and use it to calculate the area.
+        local geom = object:as_polygon()
+        tables.areas:insert({
+            geom = geom,
+            area = geom:spherical_area()  -- calculate "real" area in spheroid
+        })
+    else
+        -- modif : on enregistre la géométrie directement en multilinestring
+        -- en mergeant les lignes le plus possible
+        tables.lines:insert({
+            geom = object:as_multilinestring():line_merge()
+        })
+    end
+end
+
+function osm2pgsql.process_relation(object)
+    if clean_tags(object.tags) then
+        return
+    end
+
+    local relation_type = object:grab_tag('type')
+
+    -- Store route relations as multilinestrings
+    if relation_type == 'route' or relation_type == 'associatedStreet' or relation_type == 'public_transport' or relation_type == 'waterway' then
+        tables.lines:insert({
+            geom = object:as_multilinestring():line_merge()
+        })
+        return
+    end
+
+    -- Store multipolygon and boundary relations as polygons
+    if relation_type == 'multipolygon' or relation_type == 'boundary' then
+        local geom = object:as_multipolygon()
+        tables.areas:insert({
+            geom = geom,
+            area = geom:spherical_area()
+        })
+    end
+end
+

backends/postgres_osm2pgsql/postgres_osm2pgsql.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'pg'
+require 'overpass_parser/sql_dialect/postgres'
+
+class PostgresOsm2pgsql
+  def initialize
+    
+    @@con = PG.connect(ENV['DATABASE_URL'])
+    @@con.query(File.read(File.dirname(__FILE__) + '/view.sql'))
+    @dialect = OverpassParser::SqlDialect::Postgres.new(postgres_escape_literal: ->(s) { @@con.escape_literal(s) })
+  end
+
+  def exec(query)
+    request = OverpassParser.parse(query)
+    sql = request.to_sql(@dialect)
+    puts sql
+    result = @@con.exec(sql)
+    [sql, result.collect { |row| row['j'].gsub('+00:00', 'Z') }]
+  end
+end