Run jobs on self-hosted runners

.circleci/config.yml

@@ -2,55 +2,59 @@ version: 2.1
 
 executors:
   python-container:
+    parameters:
+      python_version:
+        type: string
+        default: "3.14"
     docker:
-      - image: cimg/python:3.12
+      - image: cimg/python:<< parameters.python_version >>
     resource_class: arangodb/small-amd64
-  python-vm:
-    machine:
-      image: ubuntu-2404:current
-    resource_class: medium
+  arangodb-container:
+    parameters:
+      arangodb_image:
+        type: string
+    docker:
+      - image: << parameters.arangodb_image >>
+        entrypoint: /bin/sh
+        command: -c "while true; do sleep 10; done"
+        user: root
+    resource_class: arangodb/medium-amd64
 
 workflows:
   ci:
     jobs:
       - lint
-      - test:
-          name: Python (<< matrix.python_version >>) - ArangoDB (enterprise, << matrix.arangodb_version >> << matrix.arangodb_config >>)
+      - import-test:
+          name: Imports (Python << matrix.python_version >>)
           matrix:
             parameters:
-              python_version: ["3.13"]
-              arangodb_config: ["single", "cluster"]
-              arangodb_license: ["enterprise"]
-              arangodb_version: ["3.12"]
-      - test:
-          name: Python (<< matrix.python_version >>) - ArangoDB (enterprise-preview, << matrix.arangodb_version >> << matrix.arangodb_config >>)
+              python_version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+      - integration-test:
+          name: ArangoDB (enterprise, 3.12 << matrix.arangodb_config >>)
           matrix:
             parameters:
-              python_version: ["3.13"]
               arangodb_config: ["single", "cluster"]
-              arangodb_license: ["enterprise-preview"]
-              arangodb_version: ["4.0-nightly"]
-      - test:
-          name: Python (<< matrix.python_version >>) - ArangoDB (enterprise, 3.12 cluster)
+              arangodb_image: ["arangodb/enterprise:3.12"]
+              arangodb_conf_version: ["3.12"]
+      - integration-test:
+          name: ArangoDB (enterprise-preview, 4.0-nightly << matrix.arangodb_config >>)
           matrix:
             parameters:
-              python_version: ["3.10", "3.11", "3.12"]
-              arangodb_config: ["cluster"]
-              arangodb_license: ["enterprise"]
-              arangodb_version: ["3.12"]
+              arangodb_config: ["single", "cluster"]
+              arangodb_image: ["arangodb/enterprise-preview:4.0-nightly"]
+              arangodb_conf_version: ["4.0"]
 
 jobs:
   lint:
     executor: python-container
-    resource_class: arangodb/small-amd64
     steps:
       - checkout
       - run:
           name: Install Dependencies
-          command: pip install .[dev]
+          command: python -m pip install .[dev]
       - run:
-            name: Run black
-            command: black --check --verbose --diff --color --config=pyproject.toml ./arango ./tests/
+          name: Run black
+          command: black --check --verbose --diff --color --config=pyproject.toml ./arango ./tests/
       - run:
           name: Run flake8
           command: flake8 ./arango ./tests
@@ -60,58 +64,137 @@ jobs:
       - run:
           name: Run mypy
           command: mypy ./arango
-  test:
+  import-test:
+    parameters:
+      python_version:
+        type: string
+    executor:
+      name: python-container
+      python_version: << parameters.python_version >>
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python --version
+            python -m pip install .[dev]
+      - run:
+          name: Run Import Tests
+          command: python -m pytest tests/test_imports.py --skip-arango-setup --color=yes --code-highlight=yes
+  integration-test:
     parameters:
-        python_version:
-            type: string
-        arangodb_config:
-            type: string
-        arangodb_license:
-            type: string
-        arangodb_version:
-            type: string
-    executor: python-vm
+      arangodb_config:
+        type: string
+      arangodb_image:
+        type: string
+      arangodb_conf_version:
+        type: string
+    executor:
+      name: arangodb-container
+      arangodb_image: << parameters.arangodb_image >>
     steps:
+      - run:
+          name: Bootstrap CI Tools
+          command: |
+            apk add --no-cache \
+              bash \
+              build-base \
+              ca-certificates \
+              curl \
+              git \
+              jq \
+              openssh-client \
+              openssl-dev \
+              py3-pip \
+              py3-virtualenv \
+              python3 \
+              python3-dev
       - checkout
       - run:
           name: Setup ArangoDB
           command: |
-            chmod +x starter.sh
-            ./starter.sh << parameters.arangodb_config >> << parameters.arangodb_license >> << parameters.arangodb_version >>
-      - restore_cache:
-          key: pip-and-local-cache
+            mkdir -p /tests
+            if [ ! -e /tests/static ]; then
+              ln -s "$PWD/tests/static" /tests/static
+            fi
+
+            conf_file="/tests/static/<< parameters.arangodb_config >>-<< parameters.arangodb_conf_version >>.conf"
+            if [ ! -f "$conf_file" ]; then
+              echo "Missing configuration file: $conf_file"
+              exit 1
+            fi
+
+            arangodb --configuration="$conf_file" > arangodb.log 2>&1 &
+            echo "$!" > arangodb.pid
+
+            ports="8529"
+            if [ << parameters.arangodb_config >> = "cluster" ]; then
+              ports="$ports 8539 8549"
+            fi
+
+            for port in $ports; do
+              echo "Waiting for ArangoDB at http://localhost:$port/_api/version"
+              for attempt in $(seq 1 120); do
+                if curl --fail --silent --show-error \
+                    --user root:passwd \
+                    "http://localhost:$port/_api/version" | jq; then
+                  break
+                fi
+
+                if ! kill -0 "$(cat arangodb.pid)" 2>/dev/null; then
+                  echo "ArangoDB process exited before port $port became ready"
+                  exit 1
+                fi
+
+                if [ "$attempt" -eq 120 ]; then
+                  echo "Timed out waiting for ArangoDB on port $port"
+                  exit 1
+                fi
+
+                sleep 1
+              done
+            done
+      - run:
+          name: Show ArangoDB Logs
+          command: tail -n 1000 arangodb.log
+          when: always
       - run:
           name: Setup Python
+          shell: /bin/bash -eo pipefail
           command: |
-            pyenv --version
-            pyenv install -f << parameters.python_version >>
-            pyenv global << parameters.python_version >>
+            python3 --version
+            python3 -m virtualenv .venv
+            source .venv/bin/activate
+            python --version
+            python -m pip --version
       - run:
           name: Install Dependencies
-          command: pip install -e .[dev]
-      - run: docker ps -a
-      - run: docker logs arango
+          shell: /bin/bash -eo pipefail
+          command: |
+            source .venv/bin/activate
+            python -m pip install -e .[dev]
       - run:
           name: Run pytest
+          shell: /bin/bash -eo pipefail
           command: |
             mkdir test-results
             mkdir htmlcov
 
+            source .venv/bin/activate
+
             args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost" "--port=8529")
             if [ << parameters.arangodb_config >> = "cluster" ]; then
               args+=("--cluster" "--port=8539" "--port=8549")
             fi
 
-            if [ << parameters.arangodb_license >> != "enterprise" ] && [ << parameters.arangodb_license >> != "enterprise-preview" ]; then
-              args+=("--skip" "enterprise")
-            fi
-
             echo "Running pytest with args: ${args[@]}"
-            pytest --cov=arango --cov-report=html:htmlcov --color=yes --code-highlight=yes "${args[@]}"
+            pytest tests --cov=arango --cov-report=html:htmlcov --color=yes --code-highlight=yes "${args[@]}"
       - store_artifacts:
           path: htmlcov
           destination: coverage-report
       - store_artifacts:
           path: test-results
+      - store_artifacts:
+          path: arangodb.log
       - store_test_results:
           path: test-results

ci/conftest.py

@@ -0,0 +1 @@
+from tests.conftest import *  # noqa: F401,F403

tests/conftest.py

@@ -48,6 +48,7 @@ class GlobalData:
     db_version: version = version.parse("0.0.0")
     is_instrumented: bool = False
     crash: bool = False
+    skip_arango_setup: bool = False
 
 
 global_data = GlobalData()
@@ -111,9 +112,18 @@ def pytest_addoption(parser):
         default=[],
         help="Skip specific tests",
     )
+    parser.addoption(
+        "--skip-arango-setup",
+        action="store_true",
+        help="Skip the ArangoDB connection check during pytest configuration",
+    )
 
 
 def pytest_configure(config):
+    if config.getoption("skip_arango_setup"):
+        global_data.skip_arango_setup = True
+        return
+
     ports = config.getoption("port")
     if ports is None:
         ports = ["8529"]

tests/test_imports.py

@@ -0,0 +1,21 @@
+import importlib
+import pkgutil
+
+import pytest
+
+import arango
+
+
+def arango_modules():
+    yield arango.__name__
+
+    for module_info in pkgutil.walk_packages(
+        arango.__path__,
+        prefix=f"{arango.__name__}.",
+    ):
+        yield module_info.name
+
+
+@pytest.mark.parametrize("module_name", sorted(arango_modules()))
+def test_import_module(module_name):
+    importlib.import_module(module_name)