Merge branch 'geopython:master' into deploy

Author: sjordan29

.github/workflows/main.yml

@@ -24,8 +24,26 @@ jobs:
           - python-version: '3.10'
     env:
       PYGEOAPI_CONFIG: "$(pwd)/pygeoapi-config.yml"
+
+    services:
+      postgres:
+        image: postgis/postgis:14-3.2
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_DB: test
+          POSTGRES_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }}
 
     steps:
+    - name: Pre-pull Docker Images
+      run: |
+        docker pull container-registry.oracle.com/database/express:21.3.0-xe &
+        docker pull appropriate/curl:latest &
+        docker pull elasticsearch:8.17.0 &
+        docker pull opensearchproject/opensearch:2.18.0 &
+        docker pull mongo:8.0.4 &
+        docker pull ghcr.io/cgs-earth/sensorthings-action:0.1.0 &
+        docker pull postgis/postgis:14-3.2 &
     - name: Clear up GitHub runner diskspace
       run: |
         echo "Space before"
@@ -48,11 +66,16 @@ jobs:
         sudo sysctl -w vm.swappiness=1
         sudo sysctl -w fs.file-max=262144
         sudo sysctl -w vm.max_map_count=262144
-    - name: Install and run PostgreSQL/PostGIS 📦
-      uses: huaxk/postgis-action@v1
+    - name: "Install and run MySQL 📦"
+      uses: mirromutth/mysql-action@v1.1
       with:
-        postgresql password: ${{ secrets.DatabasePassword || 'postgres' }}
-        postgresql db: 'test'
+        host port: 3306 
+        mysql version: '8.0' 
+        mysql database: test_geo_app
+        mysql root password: mysql # This is a dummy password here; not actually used in prod
+        mysql user: pygeoapi 
+        mysql password: mysql 
+        
     - name: Install and run Elasticsearch 📦
       uses: getong/elasticsearch-action@v1.2
       with:
@@ -111,6 +134,7 @@ jobs:
         psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_data.sql
         psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_types_data.sql
         psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/postgres_manager_full_structure.backup.sql
+        mysql -h 127.0.0.1 -P 3306 -u root -p'mysql' test_geo_app < tests/data/mysql_data.sql
         docker ps
         python3 tests/load_oracle_data.py
     - name: run unit tests ⚙️
@@ -119,6 +143,7 @@ jobs:
       run: |
         pytest tests/api
         pytest tests/test_api_ogr_provider.py
+        pytest tests/test_base_provider.py
         pytest tests/test_config.py
         pytest tests/test_csv__formatter.py
         pytest tests/test_csv__provider.py
@@ -142,6 +167,8 @@ jobs:
         pytest tests/test_oracle_provider.py
         pytest tests/test_parquet_provider.py
         pytest tests/test_postgresql_provider.py
+        pytest tests/test_postgresql_mvt_provider.py
+        pytest tests/test_mysql_provider.py
         pytest tests/test_rasterio_provider.py
         pytest tests/test_sensorthings_edr_provider.py 
         pytest tests/test_sensorthings_provider.py

CONTRIBUTING.md

@@ -65,6 +65,7 @@ Your contribution will be under our [license](https://github.com/geopython/pygeo
 * Pull requests may include copyright in the source code header by the contributor if the contribution is significant or the contributor wants to claim copyright on their contribution.
 * All contributors shall be listed at https://github.com/geopython/pygeoapi/graphs/contributors
 * Unclaimed copyright, by default, is assigned to the main copyright holders as specified in https://github.com/geopython/pygeoapi/blob/master/LICENSE.md
+* further notes can be found in the [documentation](https://docs.pygeoapi.io/en/latest/development.html#testing)
 
 ### Version Control Branching
 
@@ -89,16 +90,23 @@ Your contribution will be under our [license](https://github.com/geopython/pygeo
       while, __make sure you rebase or merge to latest ``master``__ to ensure a
         speedier resolution.
 
+### Testing
+
+* testing is performed using [pytest](https://pytest.org) and covers all major components (API core, providers, formatters, managers, etc.)
+* additional information can be found in the [documentation](https://docs.pygeoapi.io/en/latest/development.html#testing)
+
+
 ### Documentation
 
 * documentation is managed in `docs/`, in reStructuredText format
 * [Sphinx](https://www.sphinx-doc.org) is used to generate the documentation
 * See the [reStructuredText Primer](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) on rST markup and syntax
+* additional information can be found in the [documentation](https://docs.pygeoapi.io/en/latest/development.html#building-the-documentation)
 
 
 ### Wiki
 
-* the [pygeoapi wiki](https://github.com/geopython/pygeoapi/wiki) provides working level documentation
+* the [pygeoapi wiki](https://github.com/geopython/pygeoapi/wiki) provides working level documentation and various community resources
 
 
 ### Code Formatting
@@ -107,7 +115,8 @@ Your contribution will be under our [license](https://github.com/geopython/pygeo
 * pygeoapi follows the [PEP-8](http://www.python.org/dev/peps/pep-0008/) guidelines
 * 80 characters
 * spaces, not tabs
-* pygeoapi, instead of PyGeoAPI, pygeoAPI, etc.
+* pygeoapi, NOT PyGeoAPI, pygeoAPI, etc.
+* additional information can be found in the [documentation](https://docs.pygeoapi.io/en/latest/development.html#linting)
 
 ## Suggesting Enhancements
 

Dockerfile

@@ -69,6 +69,7 @@ ARG ADD_DEB_PACKAGES="\
     python3-netcdf4 \
     python3-pandas \
     python3-psycopg2 \
+    python3-pydantic \
     python3-pymongo \
     python3-pyproj \
     python3-rasterio \

docs/source/configuration.rst

@@ -42,6 +42,8 @@ For more information related to API design rules (the ``api_rules`` property in
         host: 0.0.0.0  # listening address for incoming connections
         port: 5000  # listening port for incoming connections
     url: http://localhost:5000/  # url of server
+    icon: https://example.org/favicon.ico  # favicon / shortcut icon for default HTML template customization
+    logo: https://example.org/logo.png  # logo/banner for default HTML template customization
     mimetype: application/json; charset=UTF-8  # default MIME type
     encoding: utf-8  # default server encoding
     language: en-US  # default server language

docs/source/contributing.rst

@@ -3,46 +3,5 @@
 Contributing
 ============
 
-Building the documentation
---------------------------
-
-To build the documentation in pygeoapi we use `Sphinx`_. The documentation is located in the docs folder.
-
-.. note::
-   For the following instructions to work, you must be located in the root folder of pygeoapi.
-
-Install the dependencies necessary for building the documentation using the following command:
-
-.. code-block:: bash
-
-   pip3 install -r docs/requirements.txt
-
-After installing the requirements, build the documentation using the ``sphinx-build`` command:
-
-.. code-block:: bash
-
-   sphinx-build -M html docs/source docs/build
-
-
-Or using the following ``make`` command:
-
-.. code-block:: bash
-
-   make -C docs html
-
-After building the documentation, the folder ``docs/build`` will contain the website generated with the documentation. 
-Add the folder to a web server or open the file ``docs/build/html/index.html`` file in a web browser to see the contents of the documentation.
-
-The documentation is hosted on `readthedocs`_. It is automatically generated from the contents of the ``master`` branch on GitHub.
-
-The file ``.readthedocs.yaml`` contains the configuration of the readthedocs build. Refer to the `readthedocs configuration file`_ documentation for more information.
-
-Contributing GitHub page
-------------------------
-
 Please see the `Contributing page <https://github.com/geopython/pygeoapi/blob/master/CONTRIBUTING.md>`_
 for information on contributing to the project.
-
-.. _`Sphinx`: https://www.djangoproject.com
-.. _`readthedocs`: https://docs.readthedocs.io/en/stable/index.html
-.. _readthedocs configuration file: https://docs.readthedocs.io/en/stable/config-file/v2.html

docs/source/crs.rst

@@ -30,7 +30,7 @@ Metadata
 The conformance class `http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs` is present as a `conformsTo` field
 in the root landing page response.
 
-The configured CRSs, or their defaults, `crs` and `storageCRS` and optionally `storageCrsCoordinateEpoch` will be present in the "Describe Collection" response.
+The configured CRSs, or their defaults, `crs` and `storageCrs` and optionally `storageCrsCoordinateEpoch` will be present in the "Describe Collection" response.
 
 Parameters
 ----------
@@ -95,7 +95,7 @@ Suppose an addresses collection with the following CRS support in its collection
     "http://www.opengis.net/def/crs/EPSG/0/28992",
     "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
    ],
-   "storageCRS": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
+   "storageCrs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
 
 
 This allows a `bbox-crs` query using Dutch "RD" coordinates with CRS `http://www.opengis.net/def/crs/EPSG/0/28992` to retrieve

docs/source/data-publishing/ogcapi-edr.rst

@@ -55,6 +55,7 @@ The `xarray-edr`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data
          # to derive automagically
          x_field: lon
          y_field: lat
+         z_field: z
          time_field: time
          # optionally specify the coordinate reference system of your dataset
          # else pygeoapi assumes it is WGS84 (EPSG:4326).

docs/source/data-publishing/ogcapi-features.rst

@@ -19,12 +19,13 @@ parameters.
    :header: Provider, property filters/display, resulttype, bbox, datetime, sortby, skipGeometry, domains, CQL, transactions, crs
    :align: left
 
-   `CSV`_,✅/✅,results/hits,❌,❌,❌,✅,❌,❌,❌,✅
+   `CSV`_,✅/✅,results/hits,✅,❌,❌,✅,❌,❌,❌,✅
    `Elasticsearch`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅,✅
    `ERDDAP Tabledap Service`_,❌/❌,results/hits,✅,✅,❌,❌,❌,❌,❌,✅
    `ESRI Feature Service`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,❌,✅
-   `GeoJSON`_,✅/✅,results/hits,❌,❌,❌,✅,❌,❌,❌,✅
+   `GeoJSON`_,✅/✅,results/hits,✅,❌,❌,✅,❌,❌,❌,✅
    `MongoDB`_,✅/❌,results,✅,✅,✅,✅,❌,❌,❌,✅
+   `MySQL`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅,✅
    `OGR`_,✅/❌,results/hits,✅,❌,❌,✅,❌,❌,❌,✅
    `OpenSearch`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅,✅
    `Oracle`_,✅/✅,results/hits,✅,❌,✅,✅,❌,❌,❌,✅
@@ -222,6 +223,76 @@ Here `test` is the name of database , `points` is the target collection name.
          data: mongodb://localhost:27017/testdb
          collection: testplaces
 
+
+.. _MySQL:
+
+MySQL
+^^^^^
+
+.. note::
+   Requires Python packages sqlalchemy, geoalchemy2 and pymysql
+
+Must have MySQL installed.
+
+.. code-block:: yaml
+
+   providers:
+       - type: feature
+         name: MySQL
+         data:
+             host: 127.0.0.1
+             port: 3306 # Default 3306 if not provided
+             dbname: test_geo_app
+             user: mysql
+             password: mysql
+             search_path: [test_geo_app] # Same as dbname
+         id_field: locationID
+         table: location
+         geom_field: locationCoordinates
+
+A number of database connection options can be also configured in the provider in order to adjust properly the sqlalchemy engine client.
+These are optional and if not specified, the default from the engine will be used. Please see also `SQLAlchemy docs <https://docs.sqlalchemy.org/en/14/core/engines.html#custom-dbapi-connect-arguments-on-connect-routines>`_.
+
+.. code-block:: yaml
+
+    providers:
+       - type: feature
+         name: MySQL
+         data:
+             host: 127.0.0.1
+             port: 3306 # Default 3306 if not provided
+             dbname: test_geo_app
+             user: mysql
+             password: mysql
+             search_path: [test_geo_app] # Same as dbname
+         options:
+             # Maximum time to wait while connecting, in seconds.
+             connect_timeout: 10
+             # Number of *milliseconds* that transmitted data may remain
+             # unacknowledged before a connection is forcibly closed.
+             tcp_user_timeout: 10000
+             # Whether client-side TCP keepalives are used. 1 = use keepalives,
+             # 0 = don't use keepalives.
+             keepalives: 1
+             # Number of seconds of inactivity after which TCP should send a
+             # keepalive message to the server.
+             keepalives_idle: 5
+             # Number of TCP keepalives that can be lost before the client's
+             # connection to the server is considered dead.
+             keepalives_count: 5
+             # Number of seconds after which a TCP keepalive message that is not
+             # acknowledged by the server should be retransmitted.
+             keepalives_interval: 1
+         id_field: locationID
+         table: location
+         geom_field: locationCoordinates
+
+This provider has support for the CQL queries as indicated in the Provider table above.
+
+.. seealso::
+  :ref:`cql` for more details on how to use Common Query Language (CQL) to filter the collection with specific queries.
+
+
 OGR
 ^^^
 

docs/source/data-publishing/ogcapi-tiles.rst

@@ -22,6 +22,7 @@ pygeoapi core tile providers are listed below, along with supported features.
    `MVT-elastic`_,✅,✅,✅,❌,❌,✅
    `MVT-proxy`_,❓,❓,❓,❓,❌,✅
    `WMTSFacade`_,✅,❌,✅,✅,✅,❌
+   `MVT-postgresql`_,✅,✅,✅,✅,❌,✅
 
 Below are specific connection examples based on supported providers.
 
@@ -130,6 +131,48 @@ Following code block shows how to configure pygeoapi to read Mapbox vector tiles
              name: pbf
              mimetype: application/vnd.mapbox-vector-tile
 
+MVT-postgresql
+^^^^^^^^^^^^^^
+
+.. note::
+   Requires Python packages sqlalchemy, geoalchemy2 and psycopg2-binary
+
+.. note::
+   Must have PostGIS installed with protobuf-c support 
+
+This provider gives support to serving tiles generated using `PostgreSQL <https://www.postgresql.org/>`_ with `PostGIS <https://postgis.net/>`_.
+The tiles are rendered on-the-fly using `ST_AsMVT <https://postgis.net/docs/ST_AsMVT.html>`_ and related methods.
+
+This code block shows how to configure pygeoapi to render Mapbox vector tiles from a PostGIS table.
+
+.. code-block:: yaml
+
+   providers:
+       - type: tile
+         name: MVT-postgresql
+         data:
+             host: 127.0.0.1
+             port: 3010 # Default 5432 if not provided
+             dbname: test
+             user: postgres
+             password: postgres
+             search_path: [osm, public]
+         id_field: osm_id
+         table: hotosm_bdi_waterways
+         geom_field: foo_geom
+         storage_crs: http://www.opengis.net/def/crs/EPSG/0/4326
+         options:
+             zoom:
+                 min: 0
+                 max: 15
+         format:
+             name: pbf
+             mimetype: application/vnd.mapbox-vector-tile
+
+.. tip::
+   Geometry must have correctly defined :ref:`storage_crs<crs>`
+
+PostgreSQL-related connection options can also be added to `options`. Please refer to the :ref:`PostgreSQL OGC Features Provider<PostgreSQL>` documentation for more information.
 
 WMTSFacade
 ^^^^^^^^^^