Merge branch 'geopython:master' into deploy

Author: ptomasula

.github/workflows/main.yml

@@ -124,7 +124,6 @@ jobs:
         pip3 install -r requirements-django.txt
         python3 setup.py install
         pip3 install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version`
-        #pip3 install --upgrade rasterio==1.1.8
     - name: setup test data ⚙️
       run: |
         python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid
@@ -137,50 +136,20 @@ jobs:
         mysql -h 127.0.0.1 -P 3306 -u root -p'mysql' test_geo_app < tests/data/mysql_data.sql
         docker ps
         python3 tests/load_oracle_data.py
-    - name: run unit tests ⚙️
+    - name: run API tests ⚙️
+      run: pytest tests/api --ignore-glob='*_live.py'
+    - name: run Formatter tests ⚙️
+      run: pytest tests/formatter
+    - name: run Provider tests ⚙️
       env:
         POSTGRESQL_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }}
-      run: |
-        pytest tests/api
-        pytest tests/test_api_ogr_provider.py
-        pytest tests/test_base_provider.py
-        pytest tests/test_config.py
-        pytest tests/test_csv__formatter.py
-        pytest tests/test_csv__provider.py
-        pytest tests/test_django.py
-        pytest tests/test_elasticsearch__provider.py
-        pytest tests/test_opensearch__provider.py
-        pytest tests/test_esri_provider.py
-        pytest tests/test_filesystem_provider.py
-        pytest tests/test_geojson_provider.py
-        pytest tests/test_linked_data.py
-        pytest tests/test_mongo_provider.py
-        pytest tests/test_ogr_csv_provider.py
-        pytest tests/test_ogr_esrijson_provider.py
-        pytest tests/test_ogr_gpkg_provider.py
-        pytest tests/test_ogr_shapefile_provider.py
-        pytest tests/test_ogr_sqlite_provider.py
-        pytest tests/test_ogr_wfs_provider.py
-        pytest tests/test_postgresql_manager.py
-        # pytest tests/test_ogr_wfs_provider_live.py  # NOTE: these are skipped in the file but listed here for completeness
-        pytest tests/test_openapi.py
-        pytest tests/test_oracle_provider.py
-        pytest tests/test_parquet_provider.py
-        pytest tests/test_postgresql_provider.py
-        pytest tests/test_postgresql_mvt_provider.py
-        pytest tests/test_mysql_provider.py
-        pytest tests/test_rasterio_provider.py
-        pytest tests/test_sensorthings_edr_provider.py 
-        pytest tests/test_sensorthings_provider.py
-        pytest tests/test_socrata_provider.py
-        # pytest tests/test_socrata_provider_live.py.py  # NOTE: these are skipped in the file but listed here for completeness
-        pytest tests/test_sqlite_geopackage_provider.py
-        pytest tests/test_tinydb_catalogue_provider.py
-        pytest tests/test_tinydb_manager_for_parallel_requests.py
-        pytest tests/test_util.py
-        pytest tests/test_xarray_netcdf_provider.py
-        pytest tests/test_xarray_zarr_provider.py
-        pytest tests/test_admin_api.py
+      run: pytest tests/provider --ignore-glob='*_live.py'
+    - name: run Manager tests ⚙️
+      env:
+        POSTGRESQL_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }}
+      run: pytest tests/manager
+    - name: run other tests ⚙️
+      run: pytest tests/other --ignore-glob='*_live.py'
     - name: failed tests 🚩
       if: ${{ failure() }}
       run: |

Dockerfile

@@ -8,7 +8,7 @@
 # Copyright (c) 2020 Tom Kralidis
 # Copyright (c) 2019 Just van den Broecke
 # Copyright (c) 2025 Francesco Bartoli
-# Copyright (c) 2024 Angelos Tzotsos
+# Copyright (c) 2025 Angelos Tzotsos
 # Copyright (c) 2023 Bernhard Mallinger
 #
 # Permission is hereby granted, free of charge, to any person
@@ -34,7 +34,7 @@
 #
 # =================================================================
 
-FROM ubuntu:jammy-20240911.1
+FROM ubuntu:jammy-20250714
 
 LABEL maintainer="Just van den Broecke <justb4@gmail.com>"
 
@@ -140,6 +140,10 @@ RUN python3 -m pip install --no-cache-dir -e .
 RUN \
     # Set default config and entrypoint for Docker Image
     cp /pygeoapi/docker/default.config.yml /pygeoapi/local.config.yml \
-    && cp /pygeoapi/docker/entrypoint.sh /entrypoint.sh
+    && cp /pygeoapi/docker/entrypoint.sh /entrypoint.sh \
+    # compile language files
+    && cd /pygeoapi \
+    && for i in locale/*; do echo $i && pybabel compile -d locale -l `basename $i`; done
+
 
 ENTRYPOINT ["/entrypoint.sh"]

docker/entrypoint.sh

@@ -37,8 +37,13 @@ echo "START /entrypoint.sh"
 set +e
 
 export PYGEOAPI_HOME=/pygeoapi
-export PYGEOAPI_CONFIG="${PYGEOAPI_HOME}/local.config.yml"
-export PYGEOAPI_OPENAPI="${PYGEOAPI_HOME}/local.openapi.yml"
+
+if [[ -z "$PYGEOAPI_CONFIG" ]]; then
+	export PYGEOAPI_CONFIG="${PYGEOAPI_HOME}/local.config.yml"
+fi
+if [[ -z "$PYGEOAPI_OPENAPI" ]]; then
+	export PYGEOAPI_OPENAPI="${PYGEOAPI_HOME}/local.openapi.yml"
+fi
 
 # gunicorn env settings with defaults
 SCRIPT_NAME=${SCRIPT_NAME:=/}
@@ -62,6 +67,8 @@ function error() {
 # Workdir
 cd ${PYGEOAPI_HOME}
 
+echo "Default config in ${PYGEOAPI_CONFIG}"
+
 echo "Trying to generate openapi.yml"
 pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI}
 
@@ -114,8 +121,12 @@ case ${entry_cmd} in
 
 	# Run pygeoapi server with hot reload
 	run-with-hot-reload)
-		# Lock all Python files (for gunicorn hot reload)
-		find . -type f -name "*.py" | xargs chmod 0444
+		# Lock all Python files (for gunicorn hot reload), if running with user root
+		if [[ $(id -u) -eq 0 ]]
+		then
+			echo "Running pygeoapi as root"
+			find . -type f -name "*.py" | xargs chmod 0444
+		fi
 
 		# Start with hot reload options
 		start_gunicorn --reload --reload-extra-file ${PYGEOAPI_CONFIG}

docs/source/conf.py

@@ -112,7 +112,7 @@ def __getattr__(cls, name):
 # built documents.
 #
 # The short X.Y version.
-version = '0.21.dev0'
+version = '0.22.dev0'
 # The full version, including alpha/beta/rc tags.
 release = version
 

docs/source/configuration.rst

@@ -250,6 +250,7 @@ default.
                     mimetype: application/json  # required: format mimetype
                 options:  # optional options to pass to provider (i.e. GDAL creation)
                     option_name: option_value
+                include_extra_query_parameters: false  # include extra query parameters that are not part of the collection properties (default: false)
 
       hello-world:  # name of process
           type: process  # REQUIRED (collection, process, or stac-collection)

docs/source/data-publishing/ogcapi-features.rst

@@ -39,7 +39,7 @@ parameters.
 .. note::
 
    * All Providers that support `bbox` also support the `bbox-crs` parameter. `bbox-crs` is handled within pygeoapi core.
-   * All Providers support the `crs` parameter to reproject (transform) response data. Some, like PostgreSQL and OGR, perform this natively: '✅n'.
+   * All Providers support the `crs` parameter to reproject (transform) response data. Some, like PostgreSQL and OGR, perform this natively.
 
 
 Connection examples
@@ -558,12 +558,19 @@ The ``ORACLE_POOL_MIN`` and ``ORACLE_POOL_MAX`` environment variables are used t
 If none or only one of the environment variables is set, session pooling will not be activated and standalone connections are established at every request.
 
 
+Extra_params
+""""""""""""
+The Oracle provider allows for additional parameters that can be passed in the request. It allows for the processing of additional parameters that are not defined in the ``pygeoapi-config.yml`` to be passed to a custom SQL-Manipulator-Plugin. An example use case of this is advanced filtering without exposing the filtered columns like follows ``.../collections/some_data/items?is_recent=true``. The ``SqlManipulator`` plugin's ``process_query`` method would receive ``extra_params = {'is_recent': 'true'}`` and could dynamically add a custom condition to the SQL query, like ``AND SYSDATE - create_date < 30``.
+
+The ``include_extra_query_parameters`` has to be set to ``true`` for the collection in ``pygeoapi-config.yml``. This ensures that the additional request parameters (e.g. ``is_recent=true``) are not discarded. 
+
+
 Custom SQL Manipulator Plugin
 """""""""""""""""""""""""""""
 The provider supports a SQL-Manipulator-Plugin class. With this, the SQL statement could be manipulated. This is
 useful e.g. for authorization at row level or manipulation of the explain plan with hints. 
 
-An example an more information about that feature you can find in the test class in tests/test_oracle_provider.py.
+More information and examples about this feature can be found in ``tests/provider/test_oracle_provider.py``.
 
 .. _Parquet:
 
@@ -804,6 +811,32 @@ To publish a TinyDB (`see website <https://tinydb.readthedocs.io>`_) index, the
          id_field: identifier
          time_field: datetimefield
 
+.. _including-extra-query-parameters:
+
+Including extra query parameters
+--------------------------------
+
+By default, pygeoapi ignores any extra query parameters.  For example, for a given ``.../items`` query, the query key-value pair ``foo1=bar1`` (if ``foo1`` is not a valid property of a given collection) would be ignored by pygeoapi as well as the underlying provider.
+
+To include/accept extra query parameters, the ``include_extra_query_parameters`` directive can be set in provider configuration:
+
+.. code-block:: yaml
+
+   providers:
+       - type: feature
+         editable: true|false  # optional, default is false
+         name: TinyDB
+         data: /path/to/file.db
+         id_field: identifier
+         time_field: datetimefield
+         include_extra_query_parameters: true
+
+
+With the above configuration, pygeoapi will pass ``foo1=bar1`` to the underlying provider.  If the underlying provider does not have ``foo1`` as a queryable property, then an exception will be returned citing an unknown property.
+
+Extra query parameters are useful for custom providers who may wish for specific functionality to be triggered by query parameters that are not bound to a given collection's properties.
+
+
 Controlling the order of properties
 -----------------------------------
 

docs/source/data-publishing/ogcapi-processes.rst

@@ -3,6 +3,10 @@
 Publishing processes via OGC API - Processes
 ============================================
 
+.. note::
+
+   Publishing processes via pygeoapi requires knowledge and development of Python code.
+
 `OGC API - Processes`_ provides geospatial data processing functionality in a standards-based
 fashion (inputs, outputs).
 
@@ -37,7 +41,76 @@ The below configuration is an example of a process defined as part of a custom P
                # the MyProcess class must subclass from pygeoapi.process.base.BaseProcessor
                name: my_package.my_module.my_file.MyProcess
 
-See :ref:`example-custom-pygeoapi-processing-plugin` for processing plugin examples.
+Documenting process metadata
+----------------------------
+
+When defining a process, various metadata must be supplied in order to enable discovery and description
+of inputs and outputs.  The metadata is realized by a Python dictionary in a given process that is
+supplied to process initialization at runtime.
+
+Below is a sample process definition as a Python dictionary:
+
+.. code-block:: python
+
+   PROCESS_METADATA = {
+       'version': '0.2.0',  # the version of the process
+       'id': 'hello-world', # process identifier
+       'title': 'Hello World',  # process title, can also be multilingual
+       'description': 'An example process that takes a name as input, and echoes '  # process description, can also be multilingual
+                      'it back as output. Intended to demonstrate a simple '
+                      'process with a single literal input.',
+       'jobControlOptions': ['sync-execute', 'async-execute'],  # whether the process can be executed in sync or async mode
+       'keywords': ['hello world', 'example', 'echo'],  # keywords associated with the process
+       'links': [{  # a list of 1..n  # link objects relevant to the process
+           'type': 'text/html',
+           'rel': 'about',
+           'title': 'information',
+           'href': 'https://example.org/process',
+           'hreflang': 'en-US'
+       }],
+       'inputs': {  # process inputs (one key per input), structured as JSON Schema
+           'name': {
+               'title': 'Name',
+               'description': 'The name of the person or entity that you wish to'
+                              'be echoed back as an output',
+               'schema': {
+                   'type': 'string'
+               },
+               'minOccurs': 1,
+               'maxOccurs': 1,
+               'keywords': ['full name', 'personal']
+           },
+           'message': {
+               'title': 'Message',
+               'description': 'An optional message to echo as well',
+               'schema': {
+                   'type': 'string'
+               },
+               'minOccurs': 0,
+               'maxOccurs': 1,
+               'keywords': ['message']
+           }
+       },
+       'outputs': {  # outputs
+           'echo': {  # an identifier for the output
+               'title': 'Hello, world',
+               'description': 'A "hello world" echo with the name and (optional)'
+                              ' message submitted for processing',
+               'schema': {  # output definition, structured as JSON Schema
+                   'type': 'object',
+                   'contentMediaType': 'application/json'
+               }
+           }
+       },
+       'example': {  # example request payload
+           'inputs': {
+               'name': 'World',
+               'message': 'An optional message.',
+           }
+       }
+   }
+
+See :ref:`example-custom-pygeoapi-processing-plugin` for full processing plugin examples.
 
 Processing and response handling
 --------------------------------

docs/source/data-publishing/ogcapi-records.rst

@@ -18,9 +18,9 @@ parameters.
    :header: Provider, properties (filters), resulttype, q, bbox, datetime, sortby, properties (display), domains, CQL, transactions
    :align: left
 
-   `ElasticsearchCatalogue`_,✅,results/hits,✅,✅,✅,✅,✅,✅,✅
-   `TinyDBCatalogue`_,✅,results/hits,✅,✅,✅,✅,❌,✅,✅,✅,✅
-   `CSWFacade`_,✅,results/hits,✅,✅,✅,❌,❌,✅,❌,❌
+   `ElasticsearchCatalogue`_,✅,results/hits,✅,✅,✅,✅,✅,✅,✅,✅
+   `TinyDBCatalogue`_,✅,results/hits,✅,✅,✅,✅,❌,✅,✅,✅
+   `CSWFacade`_,✅,results/hits,✅,✅,✅,✅,❌,✅,❌,❌
 
 
 Below are specific connection examples based on supported providers.
@@ -96,6 +96,11 @@ CSW Record core model is supported as a baseline.
          title_field: title
 
 
+Including extra query parameters
+--------------------------------
+
+See the :ref:`publishing vector section <including-extra-query-parameters>` for more information on including extra query parameters.
+
 Metadata search examples
 ------------------------
 

docs/source/data-publishing/ogcapi-tiles.rst

@@ -49,7 +49,7 @@ This code block shows how to configure pygeoapi to read Mapbox vector tiles gene
              zoom:
                  min: 0
                  max: 5
-        # MVT-elastic always uses WebMercatorQuad tiling scheme
+        # MVT-tippecanoe always uses WebMercatorQuad tiling scheme
          format:
              name: pbf
              mimetype: application/vnd.mapbox-vector-tile
@@ -103,12 +103,12 @@ Following block shows how to configure pygeoapi to read Mapbox vector tiles from
        - type: tile
          name: MVT-proxy
          data: http://localhost:7800/public.ne_50m_admin_0_countries/{z}/{x}/{y}.mvt
-            options:
-              zoom:
-                min: 0
-                max: 15
-              schemes:
-                - WebMercatorQuad # this option is needed in the MVT-proxy provider
+         options:
+             zoom:
+                 min: 0
+                 max: 15
+             schemes:
+                 - WebMercatorQuad # this option is needed in the MVT-proxy provider
          format:
              name: pbf
              mimetype: application/vnd.mapbox-vector-tile
@@ -201,6 +201,16 @@ This code block shows how to configure pygeoapi to read map tiles from a WMTS.
                  min: 0
                  max: 20
 
+
+Providing custom Tile Matrix Set definitions
+--------------------------------------------
+
+By default, pygeoapi provides the ``WorldCRS84Quad`` and ``WebMercatorQuad`` TMS
+definitions, for tile providers to use accordingly. Additional TMS definitions
+may be added in pygeoapi's ``resources/definitions/tiles`` (for example, by adding
+TMS definition files directly, volume mapping / Docker ``COPY``, Docker Compose ``volumes``, etc.).
+
+
 Data access examples
 --------------------