Implement OGC API Features Part 4 transactions for the postgres provider (#1891)

* Implement OGC API Features Part 4 for the postgres provider

* Mark transactions as supported in PostgreSQL

Author: totycro

docs/source/data-publishing/ogcapi-features.rst

@@ -29,7 +29,7 @@ parameters.
    `OpenSearch`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅
    `Oracle`_,✅/✅,results/hits,✅,❌,✅,✅,❌,❌,✅
    `Parquet`_,✅/✅,results/hits,✅,✅,❌,✅,❌,❌,✅
-   `PostgreSQL`_,✅/✅,results/hits,✅,✅,✅,✅,✅,❌,✅
+   `PostgreSQL`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅
    `SQLiteGPKG`_,✅/❌,results/hits,✅,❌,❌,✅,❌,❌,✅
    `SensorThings API`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅
    `Socrata`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅

pygeoapi/provider/postgresql.py

@@ -6,11 +6,13 @@
 #          John A Stevenson <jostev@bgs.ac.uk>
 #          Colin Blackburn <colb@bgs.ac.uk>
 #          Francesco Bartoli <xbartolone@gmail.com>
+#          Bernhard Mallinger <bernhard.mallinger@eox.at>
 #
 # Copyright (c) 2018 Jorge Samuel Mendes de Jesus
 # Copyright (c) 2024 Tom Kralidis
 # Copyright (c) 2022 John A Stevenson and Colin Blackburn
 # Copyright (c) 2023 Francesco Bartoli
+# Copyright (c) 2024 Bernhard Mallinger
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -56,11 +58,12 @@
 
 from geoalchemy2 import Geometry  # noqa - this isn't used explicitly but is needed to process Geometry columns
 from geoalchemy2.functions import ST_MakeEnvelope
-from geoalchemy2.shape import to_shape
+from geoalchemy2.shape import to_shape, from_shape
 from pygeofilter.backends.sqlalchemy.evaluate import to_filter
 import pyproj
 import shapely
-from sqlalchemy import create_engine, MetaData, PrimaryKeyConstraint, asc, desc
+from sqlalchemy import create_engine, MetaData, PrimaryKeyConstraint, asc, \
+    desc, delete
 from sqlalchemy.engine import URL
 from sqlalchemy.exc import ConstraintColumnNotFoundError, \
     InvalidRequestError, OperationalError
@@ -69,7 +72,8 @@
 from sqlalchemy.sql.expression import and_
 
 from pygeoapi.provider.base import BaseProvider, \
-    ProviderConnectionError, ProviderQueryError, ProviderItemNotFoundError
+    ProviderConnectionError, ProviderInvalidDataError, ProviderQueryError, \
+    ProviderItemNotFoundError
 from pygeoapi.util import get_transform_from_crs
 
 
@@ -307,6 +311,65 @@ def get(self, identifier, crs_transform_spec=None, **kwargs):
 
         return feature
 
+    def create(self, item):
+        """
+        Create a new item
+
+        :param item: `dict` of new item
+
+        :returns: identifier of created item
+        """
+
+        identifier, json_data = self._load_and_prepare_item(
+            item, accept_missing_identifier=True)
+
+        new_instance = self._feature_to_sqlalchemy(json_data, identifier)
+        with Session(self._engine) as session:
+            session.add(new_instance)
+            session.commit()
+            result_id = getattr(new_instance, self.id_field)
+
+        # NOTE: need to use id from instance in case it's generated
+        return result_id
+
+    def update(self, identifier, item):
+        """
+        Updates an existing item
+
+        :param identifier: feature id
+        :param item: `dict` of partial or full item
+
+        :returns: `bool` of update result
+        """
+
+        identifier, json_data = self._load_and_prepare_item(
+            item, raise_if_exists=False)
+
+        new_instance = self._feature_to_sqlalchemy(json_data, identifier)
+        with Session(self._engine) as session:
+            session.merge(new_instance)
+            session.commit()
+
+        return True
+
+    def delete(self, identifier):
+        """
+        Deletes an existing item
+
+        :param identifier: item id
+
+        :returns: `bool` of deletion result
+        """
+        with Session(self._engine) as session:
+            id_column = getattr(self.table_model, self.id_field)
+            result = session.execute(
+                delete(self.table_model)
+                .where(id_column == identifier)
+            )
+            session.commit()
+
+        return result.rowcount > 0
+
     def _store_db_parameters(self, parameters, options):
         self.db_user = parameters.get('user')
         self.db_host = parameters.get('host')
@@ -343,6 +406,26 @@ def _sqlalchemy_to_feature(self, item, crs_transform_out=None):
 
         return feature
 
+    def _feature_to_sqlalchemy(self, json_data, identifier=None):
+        attributes = {**json_data['properties']}
+        # 'identifier' key maybe be present in geojson properties, but might
+        # not be a valid db field
+        attributes.pop('identifier', None)
+        attributes[self.geom] = from_shape(
+            shapely.geometry.shape(json_data['geometry']),
+            # NOTE: for some reason, postgis in the github action requires
+            # explicit crs information. i think it's valid to assume 4326:
+            # https://portal.ogc.org/files/108198#feature-crs
+            srid=4326
+        )
+        attributes[self.id_field] = identifier
+
+        try:
+            return self.table_model(**attributes)
+        except Exception as e:
+            LOGGER.exception('Failed to create db model')
+            raise ProviderInvalidDataError(str(e))
+
     def _get_order_by_clauses(self, sort_by, table_model):
         # Build sort_by clauses if provided
         clauses = []

tests/pygeoapi-test-config-postgresql.yml

@@ -134,6 +134,7 @@ resources:
                 user: postgres
                 password: postgres
                 search_path: [osm, public]
+            editable: true
             options:
                 # Maximum time to wait while connecting, in seconds.
                 connect_timeout: 10

tests/test_postgresql_provider.py

@@ -5,11 +5,13 @@
 #          John A Stevenson <jostev@bgs.ac.uk>
 #          Colin Blackburn <colb@bgs.ac.uk>
 #          Francesco Bartoli <xbartolone@gmail.com>
+#          Bernhard Mallinger <bernhard.mallinger@eox.at>
 #
 # Copyright (c) 2019 Just van den Broecke
 # Copyright (c) 2024 Tom Kralidis
 # Copyright (c) 2022 John A Stevenson and Colin Blackburn
 # Copyright (c) 2023 Francesco Bartoli
+# Copyright (c) 2024 Bernhard Mallinger
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -48,7 +50,8 @@
 
 from pygeoapi.api import API
 from pygeoapi.api.itemtypes import (
-    get_collection_items, get_collection_item, post_collection_items
+    get_collection_items, get_collection_item, manage_collection_item,
+    post_collection_items
 )
 from pygeoapi.provider.base import (
     ProviderConnectionError,
@@ -107,6 +110,25 @@ def config_types():
     }
 
 
+@pytest.fixture()
+def data():
+    return json.dumps({
+        'type': 'Feature',
+        'geometry': {
+            'type': 'MultiLineString',
+            'coordinates': [
+                [[100.0, 0.0], [101.0, 0.0]],
+                [[101.0, 0.0], [100.0, 1.0]],
+            ]
+        },
+        'properties': {
+            'identifier': 123,
+            'name': 'Flowy McFlow',
+            'waterway': 'river'
+        }
+    })
+
+
 @pytest.fixture()
 def openapi():
     with open(get_test_file_path('pygeoapi-test-openapi.yml')) as fh:
@@ -795,3 +817,44 @@ def test_get_collection_items_postgresql_automap_naming_conflicts(pg_api_):
     assert code == HTTPStatus.OK
     features = json.loads(response).get('features')
     assert len(features) == 0
+
+
+def test_transaction_basic_workflow(pg_api_, data):
+    # create
+    req = mock_api_request(data=data)
+    headers, code, content = manage_collection_item(
+        pg_api_, req, action='create', dataset='hot_osm_waterways')
+    assert code == HTTPStatus.CREATED
+
+    # update
+    data_parsed = json.loads(data)
+    new_name = data_parsed['properties']['name'] + ' Flow'
+    data_parsed['properties']['name'] = new_name
+    req = mock_api_request(data=json.dumps(data_parsed))
+    headers, code, content = manage_collection_item(
+        pg_api_, req, action='update', dataset='hot_osm_waterways',
+        identifier=123)
+    assert code == HTTPStatus.NO_CONTENT
+
+    # verify update
+    req = mock_api_request()
+    headers, code, content = get_collection_item(
+        pg_api_, req, 'hot_osm_waterways', 123)
+    assert json.loads(content)['properties']['name'] == new_name
+
+    # delete
+    req = mock_api_request(data=data)
+    headers, code, content = manage_collection_item(
+        pg_api_, req, action='delete', dataset='hot_osm_waterways',
+        identifier=123)
+    assert code == HTTPStatus.OK
+
+
+def test_transaction_create_handles_invalid_input_data(pg_api_, data):
+    data_parsed = json.loads(data)
+    data_parsed['properties']['invalid-column'] = 'foo'
+
+    req = mock_api_request(data=json.dumps(data_parsed))
+    headers, code, content = manage_collection_item(
+        pg_api_, req, action='create', dataset='hot_osm_waterways')
+    assert 'generic error' in content