From 5643450ae9790a52e8dc3b14c55a66901b7bfa03 Mon Sep 17 00:00:00 2001 From: Philippe Parage <69145356+pparage@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:28:41 +0200 Subject: [PATCH 1/3] Deploy Testing Platform through Dokploy with persistent existing data --- .dockerignore | 18 ++ .gitignore | 7 + api/urls.py | 2 +- deploy/Dockerfile | 26 +++ deploy/README.md | 42 ++++ deploy/__init__.py | 0 deploy/compose.yml | 61 ++++++ deploy/gunicorn-requirement.txt | 1 + deploy/nginx.conf | 23 ++ deploy/settings.py | 13 ++ deploy/start.sh | 10 + deploy/tests/__init__.py | 0 deploy/tests/test_start_guard.py | 50 +++++ landing_page/templates/landing_page.html | 2 +- legal_section/templates/privacy.html | 6 +- templates/404.html | 28 ++- templates/500.html | 31 ++- templates/base.html | 3 +- templates/nav.html | 8 +- testing/ciphers_info.py | 90 +++++++- .../0005_cspendpoint_cspreport_and_more.py | 99 +++++++++ testing/models.py | 60 ++++- testing/templates/check_email.html | 12 +- testing/templates/check_webapp.html | 26 ++- testing/templates/check_website.html | 9 +- testing/templates/create_csp_endpoint.html | 86 ++++++++ testing/templates/csp_endpoints.html | 99 +++++++++ testing/templates/manage_csp_endpoints.html | 95 ++++++++ testing/templates/view_csp_reports.html | 72 ++++++ testing/test_ipv6.py | 207 ++++++++++++++---- testing/validators.py | 31 ++- 31 files changed, 1127 insertions(+), 90 deletions(-) create mode 100644 .dockerignore create mode 100644 deploy/Dockerfile create mode 100644 deploy/README.md create mode 100644 deploy/__init__.py create mode 100644 deploy/compose.yml create mode 100644 deploy/gunicorn-requirement.txt create mode 100644 deploy/nginx.conf create mode 100644 deploy/settings.py create mode 100644 deploy/start.sh create mode 100644 deploy/tests/__init__.py create mode 100644 deploy/tests/test_start_guard.py create mode 100644 testing/migrations/0005_cspendpoint_cspreport_and_more.py create mode 100644 testing/templates/create_csp_endpoint.html create mode 100644 testing/templates/csp_endpoints.html create mode 100644 testing/templates/manage_csp_endpoints.html create mode 100644 testing/templates/view_csp_reports.html diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..5b429349 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.github +.env +.env.* +**/.env +**/.env.* +venv* +.venv* +**/__pycache__ +*.pyc +db +*.sqlite3 +**/*.sqlite3 +files +static +testing/bck +node_modules +docs/_build diff --git a/.gitignore b/.gitignore index 73a9de33..8e886ca0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,10 @@ node_modules/ poetry.lock poetry.lock* docs/_build/ +.env +.env.* +!.env.example +venv.* +.venv* +*.sqlite3 +*.sqlite3-* diff --git a/api/urls.py b/api/urls.py index 1d060d5e..e547ab8b 100644 --- a/api/urls.py +++ b/api/urls.py @@ -32,7 +32,7 @@ urlpatterns = [ path("check-auth/", CheckAuthApiView.as_view(), name="token_obtain_pair"), path("logout/", LogoutView.as_view(), name="logout"), - path("token/", LoginApiView.as_view(), name="login"), + path("token/", LoginApiView.as_view(), name="token"), path("token/refresh/", TokenRefreshView.as_view(), name="token_refresh"), path("schema/", SpectacularAPIView.as_view(), name="testing"), path( diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 00000000..e65be35b --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.13-slim-trixie@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285 AS app +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 DEBUG=0 \ + ALLOWED_HOSTS=testing.nc3.lu PKGVER=GitHub-0-gmain +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential libffi-dev libxml2-dev libxslt1-dev \ + libpango-1.0-0 libpangoft2-1.0-0 libcairo2 libgdk-pixbuf-2.0-0 \ + shared-mime-info fonts-dejavu-core iputils-ping nmap bind9-dnsutils whois \ + openssl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY requirements.txt deploy/gunicorn-requirement.txt /tmp/ +RUN pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt \ + && pip install --no-cache-dir --require-hashes --no-deps -r /tmp/gunicorn-requirement.txt +COPY . /app/ +COPY deploy/start.sh /app/start.sh +RUN chmod 755 /app/start.sh \ + && mkdir -p /app/db /app/files \ + && DEBUG=1 python manage.py collectstatic --noinput \ + && test ! -e /app/db/db.sqlite3 +ENV DJANGO_SETTINGS_MODULE=deploy.settings +USER www-data +ENTRYPOINT ["/app/start.sh"] + +FROM nginx:stable-alpine@sha256:dc5069ad14f19660b141b21236140b91656bf89bbc3e2417c70ae650cd66104c AS proxy +COPY --from=app /app/static/ /srv/testing-static/ +COPY deploy/nginx.conf /etc/nginx/conf.d/default.conf diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000..bb18e324 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,42 @@ +# Dokploy deployment + +Production is built from `NC3-LU/TestingPlatform`, branch `main`, using +`deploy/compose.yml` on `testingplatformprodvm2`. Configure the existing NC3 GitHub +provider in Dokploy and enable automatic deployment for pushes to `main`. + +Set a private `SECRET_KEY` in Dokploy. Keep it stable across deployments. +Set `TESTING_DATA_ROOT` in Dokploy to an existing absolute host directory containing +`db/db.sqlite3` and `files/`. These directories must be readable and writable by +UID/GID 33 (`www-data`). Keep this directory outside the Git checkout. Bind mounts +refuse missing host paths, startup refuses a missing database, and SQLite opens in +`mode=rw` so a missing database cannot be silently recreated. + +**Never delete the original database, database copies, uploads, or backups.** +The migration uses a separate, verified SQLite backup and a separate uploads copy. +Application rebuilds and container replacements reuse the same persistent paths. +Do not use `down -v`, volume pruning, database resets, or flush commands. +Database migrations are intentionally not run on startup. Schema changes need a +reviewed migration and a verified backup before the corresponding code is deployed. + +Apache continues to terminate HTTPS and renew the existing certificate, proxying +to `127.0.0.1:18080`. The nginx container serves static assets built from the same +source and proxies Django to `127.0.0.1:18081`. Neither container listens on a public +address. Host networking preserves IPv6 testing and the loopback-only SMTP relay. +The former mod_wsgi application must be disabled at cutover; retaining Apache as +the HTTPS proxy does not keep the legacy application running. + +The original service had no active Django Q worker. This deployment preserves that +state; enabling a worker requires reviewing the existing scheduled tasks first. + +Validate changes with: + +```sh +python -m unittest discover -s deploy/tests -v +docker compose -f deploy/compose.yml config --quiet +``` + +Before cutover, test on a separate database/uploads copy, check database integrity +and table counts, exercise non-mutating application routes, and compare uploads. +After stopping legacy writes, create a fresh final backup and production copy. +Keep the original database and source for recovery. If the new application has +accepted writes, recovery must preserve those newer writes before switching back. diff --git a/deploy/__init__.py b/deploy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deploy/compose.yml b/deploy/compose.yml new file mode 100644 index 00000000..7986321d --- /dev/null +++ b/deploy/compose.yml @@ -0,0 +1,61 @@ +services: + web: + build: + context: .. + dockerfile: deploy/Dockerfile + target: app + # Preserve host IPv6 routing and the loopback-only SMTP relay. + # Gunicorn binds only to loopback; Apache provides public HTTPS. + network_mode: host + restart: unless-stopped + init: true + stop_grace_period: 340s + environment: + DEBUG: "0" + SECRET_KEY: ${SECRET_KEY:?Set the production Django signing key in Dokploy} + ALLOWED_HOSTS: testing.nc3.lu,localhost + EMAIL_HOST: localhost + EMAIL_PORT: "25" + EMAIL_USE_TLS: "0" + volumes: + - type: bind + source: ${TESTING_DATA_ROOT:?Set TESTING_DATA_ROOT to the prepared database and uploads directory}/db + target: /app/db + bind: + create_host_path: false + - type: bind + source: ${TESTING_DATA_ROOT:?Set TESTING_DATA_ROOT to the prepared database and uploads directory}/files + target: /app/files + bind: + create_host_path: false + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:18081/', headers={'Host':'testing.nc3.lu'}), timeout=10)"] + interval: 30s + timeout: 15s + retries: 3 + start_period: 30s + logging: + driver: json-file + options: + max-size: 10m + max-file: "3" + proxy: + build: + context: .. + dockerfile: deploy/Dockerfile + target: proxy + network_mode: host + restart: unless-stopped + depends_on: + web: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "--header=Host: testing.nc3.lu", "http://127.0.0.1:18080/"] + interval: 30s + timeout: 15s + retries: 3 + logging: + driver: json-file + options: + max-size: 10m + max-file: "3" diff --git a/deploy/gunicorn-requirement.txt b/deploy/gunicorn-requirement.txt new file mode 100644 index 00000000..77d206e6 --- /dev/null +++ b/deploy/gunicorn-requirement.txt @@ -0,0 +1 @@ +gunicorn==26.2.0 --hash=sha256:bd249d0b3f7972f7432f0a6b6ff3b3ee2d129f70cd1ff6c09a9dd9e29a2b88e3 diff --git a/deploy/nginx.conf b/deploy/nginx.conf new file mode 100644 index 00000000..2771b01e --- /dev/null +++ b/deploy/nginx.conf @@ -0,0 +1,23 @@ +map $http_x_forwarded_proto $original_proto { + default $scheme; + https https; +} +server { + listen 127.0.0.1:18080; + server_name testing.nc3.lu; + client_max_body_size 100m; + location /static/ { + alias /srv/testing-static/; + access_log off; + expires 7d; + } + location / { + proxy_pass http://127.0.0.1:18081; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $original_proto; + proxy_read_timeout 340s; + proxy_connect_timeout 10s; + } +} diff --git a/deploy/settings.py b/deploy/settings.py new file mode 100644 index 00000000..e460907b --- /dev/null +++ b/deploy/settings.py @@ -0,0 +1,13 @@ +"""Settings for the existing-database deployment managed by Dokploy.""" + +from testing_platform.settings import * # noqa: F403 +from testing_platform.settings import BASE_DIR, DATABASES + +# SQLite must open an existing database, never silently create an empty one. +DATABASES = { + "default": { + **DATABASES["default"], + "NAME": (BASE_DIR / "db" / "db.sqlite3").as_uri() + "?mode=rw", + "OPTIONS": {"uri": True}, + } +} diff --git a/deploy/start.sh b/deploy/start.sh new file mode 100644 index 00000000..c5200f5b --- /dev/null +++ b/deploy/start.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")" +if [ ! -s db/db.sqlite3 ]; then + echo 'Refusing to start without the existing database copy at db/db.sqlite3' >&2 + exit 78 +fi +exec python -m gunicorn testing_platform.wsgi:application \ + --bind 127.0.0.1:18081 --workers 1 --worker-class gthread --threads 2 \ + --timeout 330 --graceful-timeout 330 --access-logfile - --error-logfile - diff --git a/deploy/tests/__init__.py b/deploy/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deploy/tests/test_start_guard.py b/deploy/tests/test_start_guard.py new file mode 100644 index 00000000..97c5860b --- /dev/null +++ b/deploy/tests/test_start_guard.py @@ -0,0 +1,50 @@ +from pathlib import Path +import os, shutil, sqlite3, subprocess, tempfile, unittest + + +class StartGuardTests(unittest.TestCase): + def test_missing_database_refuses_start_without_creating_it(self): + script = Path(__file__).resolve().parents[1] / "start.sh" + self.assertTrue(script.exists(), "Startup guard must exist before deploying") + with tempfile.TemporaryDirectory(prefix="tp-start-check-") as d: + p = Path(d) + shutil.copy2(script, p / script.name) + r = subprocess.run( + ["bash", str(p / script.name)], capture_output=True, text=True + ) + self.assertNotEqual(r.returncode, 0) + self.assertIn("existing database", r.stderr) + self.assertFalse((p / "db/db.sqlite3").exists()) + + def test_present_database_is_retained_and_start_does_not_run_migrations(self): + script = Path(__file__).resolve().parents[1] / "start.sh" + self.assertTrue(script.exists(), "Startup guard must exist before deploying") + with tempfile.TemporaryDirectory(prefix="tp-start-check-") as d: + p = Path(d) + shutil.copy2(script, p / script.name) + (p / "db").mkdir() + db = p / "db/db.sqlite3" + with sqlite3.connect(db) as c: + c.execute("CREATE TABLE retained(id INTEGER)") + before = db.read_bytes() + (p / "bin").mkdir(parents=True) + fake = p / "bin/python" + fake.write_text('#!/bin/sh\nprintf "%s\\n" "$@"\n') + fake.chmod(0o700) + r = subprocess.run( + ["bash", str(p / script.name)], + capture_output=True, + text=True, + env={ + **os.environ, + "PATH": str(p / "bin") + os.pathsep + os.environ["PATH"], + }, + ) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("gunicorn", r.stdout) + self.assertNotIn("migrate", r.stdout) + self.assertEqual(db.read_bytes(), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/landing_page/templates/landing_page.html b/landing_page/templates/landing_page.html index 99f799f1..edb7bfa4 100644 --- a/landing_page/templates/landing_page.html +++ b/landing_page/templates/landing_page.html @@ -25,7 +25,7 @@
You can find more details on the - Testing + Testing Continuum information page.
diff --git a/legal_section/templates/privacy.html b/legal_section/templates/privacy.html index 05d48f7a..6cdab1d3 100644 --- a/legal_section/templates/privacy.html +++ b/legal_section/templates/privacy.html @@ -19,7 +19,7 @@Should you have any questions or remarks regarding this Privacy Policy, do not hesitate to contact us at - legal@lhc.lu + privacy@lhc.lu
Luxembourg House of Cybersecurity g.i.e., 122 rue Adolphe Fischer, L-1521 Luxembourg
Tel: (+352) 274 00 98 601
- Email: legal@lhc.lu
In particular, you have the right to access your personal data, to obtain the updating, the adjustment and the erasure of the personal data and you can exercise the rights to restrict and to object the processing. You can exercise the rights provided by articles 15 and the following of the GDPR contacting the following email address - legal@lhc.lu. + privacy@lhc.lu. In order to avoid unlawful access to your personal data, we will request you to provide a proof of your identity. If you have any queries about this Privacy Notice or experiencing any other privacy issue, we are striving to diff --git a/templates/404.html b/templates/404.html index 350a27b6..f8a88555 100644 --- a/templates/404.html +++ b/templates/404.html @@ -2,18 +2,28 @@ {% block content %} -
The page you are looking for might have been removed, +
The page you are looking for might have been removed,
had its name changed, or is temporarily unavailable.
We've encountered an unexpected issue on our servers and our team has been notified.
+Please try again in a few moments or contact support if the problem persists.
+ +We have encountered an issue on our side and are currently working on it. - Please do come back later!
- Back To HomepageThe Sender Policy Framework (SPF) is an email validation protocol that helps detect and block email spoofing. Email spoofing is a common technique used in phishing and spam emails. SPF allows the receiving mail server to verify that incoming mail from a domain comes from a host authorized by that domain’s administrators. The list of authorized sending hosts for a domain is published in the Domain Name System (DNS) records.
+The Sender Policy Framework (SPF) is an email validation protocol that helps detect and block email spoofing. Email spoofing is a common technique used in phishing and spam emails. SPF allows the receiving mail server to verify that incoming mail from a domain comes from a host authorized by that domain's administrators. The list of authorized sending hosts for a domain is published in the Domain Name System (DNS) records.
DMARC (Domain-based Message Authentication, Reporting, and Conformance) is an email authentication protocol that provides additional protection against email spoofing and phishing attacks. It uses the Sender Policy Framework (SPF) and DomainKeys Identified Mail (DKIM) standards. DMARC enables a domain owner to specify how mail servers should handle messages from their domain that don’t pass SPF or DKIM checks. This adds an extra layer of security
+DMARC (Domain-based Message Authentication, Reporting, and Conformance) is an email authentication protocol that provides additional protection against email spoofing and phishing attacks. It uses the Sender Policy Framework (SPF) and DomainKeys Identified Mail (DKIM) standards. DMARC enables a domain owner to specify how mail servers should handle messages from their domain that don't pass SPF or DKIM checks. This adds an extra layer of security