From 5643450ae9790a52e8dc3b14c55a66901b7bfa03 Mon Sep 17 00:00:00 2001 From: Philippe Parage <69145356+pparage@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:28:41 +0200 Subject: [PATCH 1/3] Deploy Testing Platform through Dokploy with persistent existing data --- .dockerignore | 18 ++ .gitignore | 7 + api/urls.py | 2 +- deploy/Dockerfile | 26 +++ deploy/README.md | 42 ++++ deploy/__init__.py | 0 deploy/compose.yml | 61 ++++++ deploy/gunicorn-requirement.txt | 1 + deploy/nginx.conf | 23 ++ deploy/settings.py | 13 ++ deploy/start.sh | 10 + deploy/tests/__init__.py | 0 deploy/tests/test_start_guard.py | 50 +++++ landing_page/templates/landing_page.html | 2 +- legal_section/templates/privacy.html | 6 +- templates/404.html | 28 ++- templates/500.html | 31 ++- templates/base.html | 3 +- templates/nav.html | 8 +- testing/ciphers_info.py | 90 +++++++- .../0005_cspendpoint_cspreport_and_more.py | 99 +++++++++ testing/models.py | 60 ++++- testing/templates/check_email.html | 12 +- testing/templates/check_webapp.html | 26 ++- testing/templates/check_website.html | 9 +- testing/templates/create_csp_endpoint.html | 86 ++++++++ testing/templates/csp_endpoints.html | 99 +++++++++ testing/templates/manage_csp_endpoints.html | 95 ++++++++ testing/templates/view_csp_reports.html | 72 ++++++ testing/test_ipv6.py | 207 ++++++++++++++---- testing/validators.py | 31 ++- 31 files changed, 1127 insertions(+), 90 deletions(-) create mode 100644 .dockerignore create mode 100644 deploy/Dockerfile create mode 100644 deploy/README.md create mode 100644 deploy/__init__.py create mode 100644 deploy/compose.yml create mode 100644 deploy/gunicorn-requirement.txt create mode 100644 deploy/nginx.conf create mode 100644 deploy/settings.py create mode 100644 deploy/start.sh create mode 100644 deploy/tests/__init__.py create mode 100644 deploy/tests/test_start_guard.py create mode 100644 testing/migrations/0005_cspendpoint_cspreport_and_more.py create mode 100644 testing/templates/create_csp_endpoint.html create mode 100644 testing/templates/csp_endpoints.html create mode 100644 testing/templates/manage_csp_endpoints.html create mode 100644 testing/templates/view_csp_reports.html diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..5b429349 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.github +.env +.env.* +**/.env +**/.env.* +venv* +.venv* +**/__pycache__ +*.pyc +db +*.sqlite3 +**/*.sqlite3 +files +static +testing/bck +node_modules +docs/_build diff --git a/.gitignore b/.gitignore index 73a9de33..8e886ca0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,10 @@ node_modules/ poetry.lock poetry.lock* docs/_build/ +.env +.env.* +!.env.example +venv.* +.venv* +*.sqlite3 +*.sqlite3-* diff --git a/api/urls.py b/api/urls.py index 1d060d5e..e547ab8b 100644 --- a/api/urls.py +++ b/api/urls.py @@ -32,7 +32,7 @@ urlpatterns = [ path("check-auth/", CheckAuthApiView.as_view(), name="token_obtain_pair"), path("logout/", LogoutView.as_view(), name="logout"), - path("token/", LoginApiView.as_view(), name="login"), + path("token/", LoginApiView.as_view(), name="token"), path("token/refresh/", TokenRefreshView.as_view(), name="token_refresh"), path("schema/", SpectacularAPIView.as_view(), name="testing"), path( diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 00000000..e65be35b --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.13-slim-trixie@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285 AS app +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 DEBUG=0 \ + ALLOWED_HOSTS=testing.nc3.lu PKGVER=GitHub-0-gmain +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential libffi-dev libxml2-dev libxslt1-dev \ + libpango-1.0-0 libpangoft2-1.0-0 libcairo2 libgdk-pixbuf-2.0-0 \ + shared-mime-info fonts-dejavu-core iputils-ping nmap bind9-dnsutils whois \ + openssl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY requirements.txt deploy/gunicorn-requirement.txt /tmp/ +RUN pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt \ + && pip install --no-cache-dir --require-hashes --no-deps -r /tmp/gunicorn-requirement.txt +COPY . /app/ +COPY deploy/start.sh /app/start.sh +RUN chmod 755 /app/start.sh \ + && mkdir -p /app/db /app/files \ + && DEBUG=1 python manage.py collectstatic --noinput \ + && test ! -e /app/db/db.sqlite3 +ENV DJANGO_SETTINGS_MODULE=deploy.settings +USER www-data +ENTRYPOINT ["/app/start.sh"] + +FROM nginx:stable-alpine@sha256:dc5069ad14f19660b141b21236140b91656bf89bbc3e2417c70ae650cd66104c AS proxy +COPY --from=app /app/static/ /srv/testing-static/ +COPY deploy/nginx.conf /etc/nginx/conf.d/default.conf diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000..bb18e324 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,42 @@ +# Dokploy deployment + +Production is built from `NC3-LU/TestingPlatform`, branch `main`, using +`deploy/compose.yml` on `testingplatformprodvm2`. Configure the existing NC3 GitHub +provider in Dokploy and enable automatic deployment for pushes to `main`. + +Set a private `SECRET_KEY` in Dokploy. Keep it stable across deployments. +Set `TESTING_DATA_ROOT` in Dokploy to an existing absolute host directory containing +`db/db.sqlite3` and `files/`. These directories must be readable and writable by +UID/GID 33 (`www-data`). Keep this directory outside the Git checkout. Bind mounts +refuse missing host paths, startup refuses a missing database, and SQLite opens in +`mode=rw` so a missing database cannot be silently recreated. + +**Never delete the original database, database copies, uploads, or backups.** +The migration uses a separate, verified SQLite backup and a separate uploads copy. +Application rebuilds and container replacements reuse the same persistent paths. +Do not use `down -v`, volume pruning, database resets, or flush commands. +Database migrations are intentionally not run on startup. Schema changes need a +reviewed migration and a verified backup before the corresponding code is deployed. + +Apache continues to terminate HTTPS and renew the existing certificate, proxying +to `127.0.0.1:18080`. The nginx container serves static assets built from the same +source and proxies Django to `127.0.0.1:18081`. Neither container listens on a public +address. Host networking preserves IPv6 testing and the loopback-only SMTP relay. +The former mod_wsgi application must be disabled at cutover; retaining Apache as +the HTTPS proxy does not keep the legacy application running. + +The original service had no active Django Q worker. This deployment preserves that +state; enabling a worker requires reviewing the existing scheduled tasks first. + +Validate changes with: + +```sh +python -m unittest discover -s deploy/tests -v +docker compose -f deploy/compose.yml config --quiet +``` + +Before cutover, test on a separate database/uploads copy, check database integrity +and table counts, exercise non-mutating application routes, and compare uploads. +After stopping legacy writes, create a fresh final backup and production copy. +Keep the original database and source for recovery. If the new application has +accepted writes, recovery must preserve those newer writes before switching back. diff --git a/deploy/__init__.py b/deploy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deploy/compose.yml b/deploy/compose.yml new file mode 100644 index 00000000..7986321d --- /dev/null +++ b/deploy/compose.yml @@ -0,0 +1,61 @@ +services: + web: + build: + context: .. + dockerfile: deploy/Dockerfile + target: app + # Preserve host IPv6 routing and the loopback-only SMTP relay. + # Gunicorn binds only to loopback; Apache provides public HTTPS. + network_mode: host + restart: unless-stopped + init: true + stop_grace_period: 340s + environment: + DEBUG: "0" + SECRET_KEY: ${SECRET_KEY:?Set the production Django signing key in Dokploy} + ALLOWED_HOSTS: testing.nc3.lu,localhost + EMAIL_HOST: localhost + EMAIL_PORT: "25" + EMAIL_USE_TLS: "0" + volumes: + - type: bind + source: ${TESTING_DATA_ROOT:?Set TESTING_DATA_ROOT to the prepared database and uploads directory}/db + target: /app/db + bind: + create_host_path: false + - type: bind + source: ${TESTING_DATA_ROOT:?Set TESTING_DATA_ROOT to the prepared database and uploads directory}/files + target: /app/files + bind: + create_host_path: false + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:18081/', headers={'Host':'testing.nc3.lu'}), timeout=10)"] + interval: 30s + timeout: 15s + retries: 3 + start_period: 30s + logging: + driver: json-file + options: + max-size: 10m + max-file: "3" + proxy: + build: + context: .. + dockerfile: deploy/Dockerfile + target: proxy + network_mode: host + restart: unless-stopped + depends_on: + web: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "--header=Host: testing.nc3.lu", "http://127.0.0.1:18080/"] + interval: 30s + timeout: 15s + retries: 3 + logging: + driver: json-file + options: + max-size: 10m + max-file: "3" diff --git a/deploy/gunicorn-requirement.txt b/deploy/gunicorn-requirement.txt new file mode 100644 index 00000000..77d206e6 --- /dev/null +++ b/deploy/gunicorn-requirement.txt @@ -0,0 +1 @@ +gunicorn==26.2.0 --hash=sha256:bd249d0b3f7972f7432f0a6b6ff3b3ee2d129f70cd1ff6c09a9dd9e29a2b88e3 diff --git a/deploy/nginx.conf b/deploy/nginx.conf new file mode 100644 index 00000000..2771b01e --- /dev/null +++ b/deploy/nginx.conf @@ -0,0 +1,23 @@ +map $http_x_forwarded_proto $original_proto { + default $scheme; + https https; +} +server { + listen 127.0.0.1:18080; + server_name testing.nc3.lu; + client_max_body_size 100m; + location /static/ { + alias /srv/testing-static/; + access_log off; + expires 7d; + } + location / { + proxy_pass http://127.0.0.1:18081; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $original_proto; + proxy_read_timeout 340s; + proxy_connect_timeout 10s; + } +} diff --git a/deploy/settings.py b/deploy/settings.py new file mode 100644 index 00000000..e460907b --- /dev/null +++ b/deploy/settings.py @@ -0,0 +1,13 @@ +"""Settings for the existing-database deployment managed by Dokploy.""" + +from testing_platform.settings import * # noqa: F403 +from testing_platform.settings import BASE_DIR, DATABASES + +# SQLite must open an existing database, never silently create an empty one. +DATABASES = { + "default": { + **DATABASES["default"], + "NAME": (BASE_DIR / "db" / "db.sqlite3").as_uri() + "?mode=rw", + "OPTIONS": {"uri": True}, + } +} diff --git a/deploy/start.sh b/deploy/start.sh new file mode 100644 index 00000000..c5200f5b --- /dev/null +++ b/deploy/start.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")" +if [ ! -s db/db.sqlite3 ]; then + echo 'Refusing to start without the existing database copy at db/db.sqlite3' >&2 + exit 78 +fi +exec python -m gunicorn testing_platform.wsgi:application \ + --bind 127.0.0.1:18081 --workers 1 --worker-class gthread --threads 2 \ + --timeout 330 --graceful-timeout 330 --access-logfile - --error-logfile - diff --git a/deploy/tests/__init__.py b/deploy/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deploy/tests/test_start_guard.py b/deploy/tests/test_start_guard.py new file mode 100644 index 00000000..97c5860b --- /dev/null +++ b/deploy/tests/test_start_guard.py @@ -0,0 +1,50 @@ +from pathlib import Path +import os, shutil, sqlite3, subprocess, tempfile, unittest + + +class StartGuardTests(unittest.TestCase): + def test_missing_database_refuses_start_without_creating_it(self): + script = Path(__file__).resolve().parents[1] / "start.sh" + self.assertTrue(script.exists(), "Startup guard must exist before deploying") + with tempfile.TemporaryDirectory(prefix="tp-start-check-") as d: + p = Path(d) + shutil.copy2(script, p / script.name) + r = subprocess.run( + ["bash", str(p / script.name)], capture_output=True, text=True + ) + self.assertNotEqual(r.returncode, 0) + self.assertIn("existing database", r.stderr) + self.assertFalse((p / "db/db.sqlite3").exists()) + + def test_present_database_is_retained_and_start_does_not_run_migrations(self): + script = Path(__file__).resolve().parents[1] / "start.sh" + self.assertTrue(script.exists(), "Startup guard must exist before deploying") + with tempfile.TemporaryDirectory(prefix="tp-start-check-") as d: + p = Path(d) + shutil.copy2(script, p / script.name) + (p / "db").mkdir() + db = p / "db/db.sqlite3" + with sqlite3.connect(db) as c: + c.execute("CREATE TABLE retained(id INTEGER)") + before = db.read_bytes() + (p / "bin").mkdir(parents=True) + fake = p / "bin/python" + fake.write_text('#!/bin/sh\nprintf "%s\\n" "$@"\n') + fake.chmod(0o700) + r = subprocess.run( + ["bash", str(p / script.name)], + capture_output=True, + text=True, + env={ + **os.environ, + "PATH": str(p / "bin") + os.pathsep + os.environ["PATH"], + }, + ) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("gunicorn", r.stdout) + self.assertNotIn("migrate", r.stdout) + self.assertEqual(db.read_bytes(), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/landing_page/templates/landing_page.html b/landing_page/templates/landing_page.html index 99f799f1..edb7bfa4 100644 --- a/landing_page/templates/landing_page.html +++ b/landing_page/templates/landing_page.html @@ -25,7 +25,7 @@

Fostering Best Practices & non-invasive manner, focusing on discovery rather than exploiting any identified vulnerabilities.

You can find more details on the - Testing + Testing Continuum information page.

diff --git a/legal_section/templates/privacy.html b/legal_section/templates/privacy.html index 05d48f7a..6cdab1d3 100644 --- a/legal_section/templates/privacy.html +++ b/legal_section/templates/privacy.html @@ -19,7 +19,7 @@

PRIVACY STATEMENT

the website or via other appropriate means. The latest applicable version will be available on our website.

Should you have any questions or remarks regarding this Privacy Policy, do not hesitate to contact us at - legal@lhc.lu + privacy@lhc.lu

PROCESSING OF PERSONAL DATA

@@ -38,7 +38,7 @@

PROCESSING OF PERSONAL DATA

Luxembourg House of Cybersecurity g.i.e., 122 rue Adolphe Fischer, L-1521 Luxembourg
Tel: (+352) 274 00 98 601
- Email: legal@lhc.lu

+ Email: privacy@lhc.lu

WHAT PERSONAL DATA DO WE COLLECT ON OUR WEBSITE AND TO WHICH END?

@@ -113,7 +113,7 @@

SECURITY OF YOUR PERSONAL DATA

In particular, you have the right to access your personal data, to obtain the updating, the adjustment and the erasure of the personal data and you can exercise the rights to restrict and to object the processing. You can exercise the rights provided by articles 15 and the following of the GDPR contacting the following email address - legal@lhc.lu. + privacy@lhc.lu. In order to avoid unlawful access to your personal data, we will request you to provide a proof of your identity. If you have any queries about this Privacy Notice or experiencing any other privacy issue, we are striving to diff --git a/templates/404.html b/templates/404.html index 350a27b6..f8a88555 100644 --- a/templates/404.html +++ b/templates/404.html @@ -2,18 +2,28 @@ {% block content %} -

-
-
-

404

+
+
+
+

404

-

We are sorry, Page not found!

-
-

The page you are looking for might have been removed, +

We are sorry, Page not found!

+

The page you are looking for might have been removed,
had its name changed, or is temporarily unavailable.

- Back To Homepage + + + +
+

What you can try:

+
    +
  • • Check the URL for errors
  • +
  • • Return to the homepage
  • +
  • • Use the navigation menu
  • +
+
- {% endblock %} diff --git a/templates/500.html b/templates/500.html index 5d663c4a..79b5fa46 100644 --- a/templates/500.html +++ b/templates/500.html @@ -2,18 +2,29 @@ {% block content %} -
-
-
-

500

+
+
+
+

500

+
+

Server Error

+

We've encountered an unexpected issue on our servers and our team has been notified.

+

Please try again in a few moments or contact support if the problem persists.

+ + + +
+

What you can try:

+
    +
  • • Refresh the page
  • +
  • • Clear your browser cache
  • +
  • • Try again later
  • +
-

Something is definitely broken

-
-

We have encountered an issue on our side and are currently working on it. - Please do come back later!

- Back To Homepage
- {% endblock %} diff --git a/templates/base.html b/templates/base.html index bfbd93dc..c13b35dd 100644 --- a/templates/base.html +++ b/templates/base.html @@ -35,9 +35,8 @@ {% include 'nav.html' %} -{% bootstrap_messages %} -
+ {% bootstrap_messages %} {% block content %}{% endblock %}
diff --git a/templates/nav.html b/templates/nav.html index 4aac3f4e..ba986fe2 100644 --- a/templates/nav.html +++ b/templates/nav.html @@ -39,12 +39,10 @@
- {% if domain %} + {% if domain and not error %}

Overview of {{ domain }}

@@ -155,7 +157,7 @@

Vulnerability

SPF Record
-

The Sender Policy Framework (SPF) is an email validation protocol that helps detect and block email spoofing. Email spoofing is a common technique used in phishing and spam emails. SPF allows the receiving mail server to verify that incoming mail from a domain comes from a host authorized by that domain’s administrators. The list of authorized sending hosts for a domain is published in the Domain Name System (DNS) records.

+

The Sender Policy Framework (SPF) is an email validation protocol that helps detect and block email spoofing. Email spoofing is a common technique used in phishing and spam emails. SPF allows the receiving mail server to verify that incoming mail from a domain comes from a host authorized by that domain's administrators. The list of authorized sending hosts for a domain is published in the Domain Name System (DNS) records.

-

DMARC (Domain-based Message Authentication, Reporting, and Conformance) is an email authentication protocol that provides additional protection against email spoofing and phishing attacks. It uses the Sender Policy Framework (SPF) and DomainKeys Identified Mail (DKIM) standards. DMARC enables a domain owner to specify how mail servers should handle messages from their domain that don’t pass SPF or DKIM checks. This adds an extra layer of security

+

DMARC (Domain-based Message Authentication, Reporting, and Conformance) is an email authentication protocol that provides additional protection against email spoofing and phishing attacks. It uses the Sender Policy Framework (SPF) and DomainKeys Identified Mail (DKIM) standards. DMARC enables a domain owner to specify how mail servers should handle messages from their domain that don't pass SPF or DKIM checks. This adds an extra layer of security

+
+
    + {% for detail in validation_details %} +
  • {{ detail }}
  • + {% endfor %} +
+
+ {% endif %} +
{% endif %} +
Assesses web security, includin {% endif %}
- {% if domain %} + {% if domain and not error and not validation_error %} +

Overview of {{ domain }}

diff --git a/testing/templates/check_website.html b/testing/templates/check_website.html index b65c1319..52153766 100644 --- a/testing/templates/check_website.html +++ b/testing/templates/check_website.html @@ -29,7 +29,9 @@

9-point security check for webs {% csrf_token %} {% if error %} - {{ error }} + {% endif %}

- - {% include "web-test_report.html" %} + {% if domain and not error %} + {% include "web-test_report.html" %} + {% endif %}
diff --git a/testing/templates/create_csp_endpoint.html b/testing/templates/create_csp_endpoint.html new file mode 100644 index 00000000..eb9be1a4 --- /dev/null +++ b/testing/templates/create_csp_endpoint.html @@ -0,0 +1,86 @@ +{% extends "base.html" %} +{% block content %} +
+
+
+
+
+

Create CSP Report Endpoint

+
+
+ {% if error %} +
{{ error }}
+ {% endif %} + + {% if endpoint_url %} +
+

Endpoint Created Successfully!

+

Your CSP report endpoint URL is:

+
+ + +
+ +
+

Add these headers to your website's configuration:

+
+
+ Testing Mode (won't block anything): +
+ + +
+
+ +
+ Enforcement Mode: +
+ + +
+
+
+ + +
+ {% else %} + + {% csrf_token %} +
+ + +
Enter the domain that will be sending CSP reports
+
+ + + {% endif %} +
+
+
+
+
+ + +{% endblock %} diff --git a/testing/templates/csp_endpoints.html b/testing/templates/csp_endpoints.html new file mode 100644 index 00000000..128f6080 --- /dev/null +++ b/testing/templates/csp_endpoints.html @@ -0,0 +1,99 @@ +{# manage_csp_endpoints.html #} +{% extends "base.html" %} +{% block content %} +
+

Manage CSP Report Endpoints

+ Create New Endpoint + + {% if endpoints %} + + + + + + + + + + + {% for endpoint in endpoints %} + + + + + + + {% endfor %} + +
Allowed OriginEndpoint URLCreatedActions
{{ endpoint.allowed_origin }}https://testing.nc3.lu/uri-report/{{ endpoint.endpoint_uuid }}/{{ endpoint.timestamp|date:"Y-m-d H:i" }} + View Reports +
+ {% else %} +

No CSP report endpoints configured yet.

+ {% endif %} +
+{% endblock %} + +{# create_csp_endpoint.html #} +{% extends "base.html" %} +{% block content %} +
+

Create CSP Report Endpoint

+ + {% if error %} +
{{ error }}
+ {% endif %} + + {% if endpoint_url %} +
+

Endpoint Created Successfully!

+

Your CSP report endpoint URL is:

+ {{ endpoint_url }} +

Add this to your Content-Security-Policy header:

+ report-uri {{ endpoint_url }}; +
+ {% endif %} + +
+ {% csrf_token %} +
+ + + Enter the domain that will be sending CSP reports +
+ +
+
+{% endblock %} + +{# view_csp_reports.html #} +{% extends "base.html" %} +{% block content %} +
+

CSP Reports for {{ endpoint.allowed_origin }}

+ + {% if reports %} + + + + + + + + + {% for report in reports %} + + + + + {% endfor %} + +
TimestampViolation Details
{{ report.timestamp|date:"Y-m-d H:i:s" }} +
{{ report.report_data|json }}
+
+ {% else %} +

No CSP violation reports received yet.

+ {% endif %} +
+{% endblock %} diff --git a/testing/templates/manage_csp_endpoints.html b/testing/templates/manage_csp_endpoints.html new file mode 100644 index 00000000..c860a02e --- /dev/null +++ b/testing/templates/manage_csp_endpoints.html @@ -0,0 +1,95 @@ +{% extends "base.html" %} +{% block content %} +
+
+
+
+

Manage CSP Report Endpoints

+ Create New Endpoint +
+ + {% if endpoints %} +
+
+
+ + + + + + + + + + + + {% for endpoint in endpoints %} + + + + + + + + {% endfor %} + +
Allowed OriginEndpoint URLStatusCreatedActions
{{ endpoint.allowed_origin }} +
+ + +
+
+ + {{ endpoint.is_active|yesno:"Active,Inactive" }} + + {{ endpoint.created_at|date:"Y-m-d H:i" }} +
+ View Reports +
+ {% csrf_token %} + + + +
+
+ {% csrf_token %} + + + +
+
+
+
+
+
+ {% else %} +
+ No CSP report endpoints configured yet. Create one to start monitoring CSP violations. +
+ {% endif %} +
+
+
+ + +{% endblock %} diff --git a/testing/templates/view_csp_reports.html b/testing/templates/view_csp_reports.html new file mode 100644 index 00000000..3bedd880 --- /dev/null +++ b/testing/templates/view_csp_reports.html @@ -0,0 +1,72 @@ +{% extends "base.html" %} +{% block content %} +
+
+
+

CSP Violation Reports for {{ endpoint.allowed_origin }}

+ + {% if reports %} +
+
+
+ + + + + + + + + + + + {% for report in reports %} + + + + + + + + {% endfor %} + +
TimestampViolated DirectiveBlocked URIDocument URIActions
{{ report.timestamp|date:"Y-m-d H:i:s" }}{{ report.report_data.violated-directive|default:report.report_data.effective-directive }}{{ report.report_data.blocked-uri }}{{ report.report_data.document-uri }} + +
+
+
+
+ {% else %} +
No CSP violation reports received yet.
+ {% endif %} +
+
+
+ + + + + +{% endblock %} diff --git a/testing/test_ipv6.py b/testing/test_ipv6.py index 0b40b64f..cca57eab 100644 --- a/testing/test_ipv6.py +++ b/testing/test_ipv6.py @@ -1,9 +1,10 @@ import logging from typing import Any, Dict, List, Union - import dns.message import dns.rdatatype import dns.resolver +import dns.query +import dns.exception logger = logging.getLogger(__name__) @@ -11,21 +12,84 @@ def ipv6_check( domain: str, port=None ) -> Dict[str, Union[Dict[Any, Any], List[Union[str, int]], List[Any]]]: + """ + Check IPv6 connectivity for a domain. + + This function tests the IPv6 readiness of a domain by examining: + 1. Name server records and their IPv6 connectivity + 2. Domain IPv6 records and their reachability + + Args: + domain (str): The domain to check + port (int, optional): The port to use for connectivity tests + + Returns: + Dict: Results of the IPv6 check containing nameservers, + connectivity comments, and records information + """ + if not domain: + logger.error("Empty domain provided to ipv6_check") + return { + "error": "No domain provided", + "nameservers": {}, + "nameservers_comments": {"grade": "null", "comment": "No domain provided"}, + "nameservers_reachability_comments": { + "grade": "null", + "comment": "No domain provided", + }, + "records": [], + "records_v4_comments": None, + "records_v6_comments": None, + } + logger.info(f"ipv6 scan: scanning domain {domain}") results = {} + # Initialize result structures with proper default values + nameservers_comments = { + "grade": "null", + "comment": "Your domain has no name server with an IPv6 record.", + } + + nameservers_reachability_comments = { + "grade": "null", + "comment": "Your domain name servers are not reachable over IPv6.", + } + + records = [] + + records_v4_comments = None + records_v6_comments = None + # Check Name Servers connectivity: - default_resolver = dns.resolver.Resolver().nameservers[0] - logger.info(f"ipv6 scan: default resolver is {default_resolver}") - q = dns.message.make_query(domain, dns.rdatatype.NS) - ns_response = dns.query.tcp(q, default_resolver) - ns_names = [ - t.target.to_text() - for ans in ns_response.answer - for t in ans - if hasattr(t, "target") - ] - logger.info(f"ipv6 scan: {len(ns_names)} name servers in domain {domain}") + try: + default_resolver = dns.resolver.Resolver().nameservers[0] + logger.info(f"ipv6 scan: default resolver is {default_resolver}") + q = dns.message.make_query(domain, dns.rdatatype.NS) + ns_response = dns.query.tcp(q, default_resolver) + ns_names = [ + t.target.to_text() + for ans in ns_response.answer + for t in ans + if hasattr(t, "target") + ] + logger.info(f"ipv6 scan: {len(ns_names)} name servers in domain {domain}") + except ( + dns.resolver.NXDOMAIN, + dns.resolver.NoAnswer, + dns.exception.DNSException, + ) as e: + logger.error(f"Failed to query NS records for {domain}: {str(e)}") + return { + "error": f"DNS query failed: {str(e)}", + "nameservers": {}, + "nameservers_comments": nameservers_comments, + "nameservers_reachability_comments": nameservers_reachability_comments, + "records": records, + "records_v4_comments": records_v4_comments, + "records_v6_comments": records_v6_comments, + } + for ns_name in ns_names: results[ns_name] = {} logger.info(f"ipv6 scan: found NS {ns_name}") @@ -33,50 +97,103 @@ def ipv6_check( q_a = dns.message.make_query(ns_name, dns.rdatatype.A) r_a = dns.query.tcp(q_a, default_resolver, timeout=5) except dns.exception.Timeout: + logger.warning(f"Timeout querying A record for {ns_name}") + r_a = None + except dns.exception.DNSException as e: + logger.warning(f"Error querying A record for {ns_name}: {str(e)}") r_a = None + try: q_aaaa = dns.message.make_query(ns_name, dns.rdatatype.AAAA) r_aaaa = dns.query.tcp(q_aaaa, default_resolver, timeout=5) except dns.exception.Timeout: + logger.warning(f"Timeout querying AAAA record for {ns_name}") + r_aaaa = None + except dns.exception.DNSException as e: + logger.warning(f"Error querying AAAA record for {ns_name}: {str(e)}") r_aaaa = None - if r_a.answer: - ns_ip4 = [item.address for answer in r_a.answer for item in answer.items][0] - q4 = dns.message.make_query("example.com", dns.rdatatype.A) - logger.info(f"{ns_name} - {ns_ip4}") - tcp4_response_default = dns.query.tcp(q4, default_resolver, timeout=5) - logger.info(f"Default resolver answer: {tcp4_response_default.answer}") + # Process A records if available + if r_a and hasattr(r_a, "answer") and r_a.answer: try: - tcp4_response = dns.query.tcp(q4, ns_ip4, timeout=5) - logger.info(f"Name server answer: {tcp4_response.answer}") - except dns.exception.Timeout: - tcp4_response = None + ns_ip4 = [ + item.address for answer in r_a.answer for item in answer.items + ][0] + q4 = dns.message.make_query("example.com", dns.rdatatype.A) + logger.info(f"{ns_name} - {ns_ip4}") + tcp4_response_default = dns.query.tcp(q4, default_resolver, timeout=5) + logger.info(f"Default resolver answer: {tcp4_response_default.answer}") + try: + tcp4_response = dns.query.tcp(q4, ns_ip4, timeout=5) + logger.info(f"Name server answer: {tcp4_response.answer}") + except dns.exception.Timeout: + logger.warning(f"Timeout querying IPv4 nameserver {ns_ip4}") + tcp4_response = None + except Exception as e: + logger.warning(f"Error querying IPv4 nameserver {ns_ip4}: {str(e)}") + tcp4_response = None + except (IndexError, AttributeError) as e: + logger.warning(f"Error processing A record for {ns_name}: {str(e)}") + ns_ip4 = None else: ns_ip4 = None - if r_aaaa.answer: - ns_ip6 = [ - item.address for answer in r_aaaa.answer for item in answer.items - ][0] - q6 = dns.message.make_query("example.com", dns.rdatatype.AAAA) - logger.info(f"{ns_name} - {ns_ip6}") - tcp6_response_default = dns.query.tcp(q6, default_resolver, timeout=5) - logger.info(f"Default resolver answer: {tcp6_response_default.answer}") + # Process AAAA records if available + if r_aaaa and hasattr(r_aaaa, "answer") and r_aaaa.answer: try: - tcp6_response = dns.query.tcp(q6, ns_ip6, timeout=5) - logger.info(f"Name server answer: {tcp6_response.answer}") - except OSError: + ns_ip6 = [ + item.address for answer in r_aaaa.answer for item in answer.items + ][0] + q6 = dns.message.make_query("example.com", dns.rdatatype.AAAA) + logger.info(f"{ns_name} - {ns_ip6}") + tcp6_response_default = dns.query.tcp(q6, default_resolver, timeout=5) + logger.info(f"Default resolver answer: {tcp6_response_default.answer}") try: - tcp6_response = dns.query.tcp(q6, ns_ip4, timeout=5) + tcp6_response = dns.query.tcp(q6, ns_ip6, timeout=5) logger.info(f"Name server answer: {tcp6_response.answer}") - except dns.exception.Timeout: - tcp6_response = None - - return { - "nameservers": results, - "nameservers_comments": nameservers_comments, - "nameservers_reachability_comments": nameservers_reachability_comments, - "records": records, - "records_v4_comments": records_v4_comments, - "records_v6_comments": records_v6_comments, - } + except OSError as e: + logger.warning( + f"OS error querying IPv6 nameserver {ns_ip6}: {str(e)}" + ) + try: + # Fall back to IPv4 if IPv6 fails + if ns_ip4: + tcp6_response = dns.query.tcp(q6, ns_ip4, timeout=5) + logger.info( + f"Name server answer (IPv4 fallback): {tcp6_response.answer}" + ) + else: + tcp6_response = None + except dns.exception.Timeout: + logger.warning(f"Timeout on IPv4 fallback for {ns_name}") + tcp6_response = None + except Exception as e: + logger.warning( + f"Error on IPv4 fallback for {ns_name}: {str(e)}" + ) + tcp6_response = None + except (IndexError, AttributeError) as e: + logger.warning(f"Error processing AAAA record for {ns_name}: {str(e)}") + tcp6_response = None + + # Build the final result structure with proper error handling + try: + return { + "nameservers": results, + "nameservers_comments": nameservers_comments, + "nameservers_reachability_comments": nameservers_reachability_comments, + "records": records, + "records_v4_comments": records_v4_comments, + "records_v6_comments": records_v6_comments, + } + except Exception as e: + logger.error(f"Error building ipv6_check results: {str(e)}") + return { + "error": f"Error processing results: {str(e)}", + "nameservers": {}, + "nameservers_comments": nameservers_comments, + "nameservers_reachability_comments": nameservers_reachability_comments, + "records": [], + "records_v4_comments": None, + "records_v6_comments": None, + } diff --git a/testing/validators.py b/testing/validators.py index 2907e461..39b6b9ba 100644 --- a/testing/validators.py +++ b/testing/validators.py @@ -17,8 +17,37 @@ def file_size(file: BytesIO): def full_domain_validator(value): + """ + Validates that a string is a valid domain name. + + Args: + value (str): The domain name to validate + + Returns: + str: The validated domain name + + Raises: + Exception: If the domain name is invalid + """ + if not value: + raise Exception("Domain name cannot be empty.") + + if not isinstance(value, str): + raise Exception("Domain name must be a string.") + + # Remove any leading/trailing whitespace + value = value.strip() + + # Check for common invalid characters + invalid_chars = ["<", ">", '"', "'", "\\", " "] + for char in invalid_chars: + if char in value: + raise Exception(f"Domain name contains invalid character: '{char}'") + res = pattern.match(value) if res: return value else: - raise Exception("This field must be a domain name.") + raise Exception( + "Invalid domain name format. Please enter a valid domain (e.g., example.com)." + ) From b9330924bf03bfa2e7d73161df3ab92d0ef2b90d Mon Sep 17 00:00:00 2001 From: Philippe Parage <69145356+pparage@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:32:41 +0200 Subject: [PATCH 2/3] Preserve the existing SMTP configuration in Dokploy --- deploy/README.md | 4 +++- deploy/compose.yml | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index bb18e324..d590a6b2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -21,10 +21,12 @@ reviewed migration and a verified backup before the corresponding code is deploy Apache continues to terminate HTTPS and renew the existing certificate, proxying to `127.0.0.1:18080`. The nginx container serves static assets built from the same source and proxies Django to `127.0.0.1:18081`. Neither container listens on a public -address. Host networking preserves IPv6 testing and the loopback-only SMTP relay. +address. Host networking preserves IPv6 testing and access to local services when needed. The former mod_wsgi application must be disabled at cutover; retaining Apache as the HTTPS proxy does not keep the legacy application running. +Mail settings (`EMAIL_HOST`, `EMAIL_PORT`, `EMAIL_USE_TLS`, `EMAIL_HOST_USER`, +`EMAIL_HOST_PASSWORD`, `DEFAULT_FROM_EMAIL`) are supplied privately in Dokploy. The original service had no active Django Q worker. This deployment preserves that state; enabling a worker requires reviewing the existing scheduled tasks first. diff --git a/deploy/compose.yml b/deploy/compose.yml index 7986321d..d44aa25d 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -4,7 +4,7 @@ services: context: .. dockerfile: deploy/Dockerfile target: app - # Preserve host IPv6 routing and the loopback-only SMTP relay. + # Preserve the existing host IPv6 routing used by network tests. # Gunicorn binds only to loopback; Apache provides public HTTPS. network_mode: host restart: unless-stopped @@ -14,9 +14,12 @@ services: DEBUG: "0" SECRET_KEY: ${SECRET_KEY:?Set the production Django signing key in Dokploy} ALLOWED_HOSTS: testing.nc3.lu,localhost - EMAIL_HOST: localhost - EMAIL_PORT: "25" - EMAIL_USE_TLS: "0" + EMAIL_HOST: ${EMAIL_HOST:-localhost} + EMAIL_PORT: ${EMAIL_PORT:-25} + EMAIL_USE_TLS: ${EMAIL_USE_TLS:-0} + EMAIL_HOST_USER: ${EMAIL_HOST_USER:-} + EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD:-} + DEFAULT_FROM_EMAIL: ${DEFAULT_FROM_EMAIL:-webmaster@localhost} volumes: - type: bind source: ${TESTING_DATA_ROOT:?Set TESTING_DATA_ROOT to the prepared database and uploads directory}/db From 03704f9eb215538c39e2a79cd845f388774e52a4 Mon Sep 17 00:00:00 2001 From: Philippe Parage <69145356+pparage@users.noreply.github.com> Date: Mon, 14 Sep 2026 15:09:55 +0200 Subject: [PATCH 3/3] Route Dokploy deployment through Traefik with a private app socket --- deploy/Dockerfile | 1 + deploy/README.md | 21 +++++++++++++------- deploy/compose.yml | 21 ++++++++++++++++---- deploy/nginx.conf | 7 +++++-- deploy/settings.py | 4 ++++ deploy/start.sh | 3 ++- deploy/tests/test_proxy_scheme.py | 33 +++++++++++++++++++++++++++++++ deploy/tests/test_start_guard.py | 1 + 8 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 deploy/tests/test_proxy_scheme.py diff --git a/deploy/Dockerfile b/deploy/Dockerfile index e65be35b..ca47052f 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -15,6 +15,7 @@ COPY . /app/ COPY deploy/start.sh /app/start.sh RUN chmod 755 /app/start.sh \ && mkdir -p /app/db /app/files \ + && mkdir -p /run/testingplatform && chown www-data:www-data /run/testingplatform \ && DEBUG=1 python manage.py collectstatic --noinput \ && test ! -e /app/db/db.sqlite3 ENV DJANGO_SETTINGS_MODULE=deploy.settings diff --git a/deploy/README.md b/deploy/README.md index d590a6b2..4d928f3a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -18,12 +18,19 @@ Do not use `down -v`, volume pruning, database resets, or flush commands. Database migrations are intentionally not run on startup. Schema changes need a reviewed migration and a verified backup before the corresponding code is deployed. -Apache continues to terminate HTTPS and renew the existing certificate, proxying -to `127.0.0.1:18080`. The nginx container serves static assets built from the same -source and proxies Django to `127.0.0.1:18081`. Neither container listens on a public -address. Host networking preserves IPv6 testing and access to local services when needed. -The former mod_wsgi application must be disabled at cutover; retaining Apache as -the HTTPS proxy does not keep the legacy application running. +Traefik on the Dokploy remote terminates HTTPS and manages certificate renewal. +In the Compose service's Dokploy Domains settings, route `testing.nc3.lu`, path +`/`, to service `proxy`, port `80`, with HTTPS and the `letsencrypt` resolver. +The nginx container joins `dokploy-network`, serves the built static assets and +forwards Django requests over the private `runtime` volume's Unix socket. +Its loopback-only `127.0.0.1:18080` mapping supports local health checks. + +The Django container retains host networking for IPv6 network tests and local +SMTP access, but Gunicorn has no TCP listener. Only the web and nginx containers +mount the socket volume. Traefik sets `X-Forwarded-Proto`, nginx preserves it, +and the deployment settings recognize public HTTPS for generated links and CSRF. +Apache and its former mod_wsgi application are stopped and disabled at cutover; +neither is part of the production request path afterward. Mail settings (`EMAIL_HOST`, `EMAIL_PORT`, `EMAIL_USE_TLS`, `EMAIL_HOST_USER`, `EMAIL_HOST_PASSWORD`, `DEFAULT_FROM_EMAIL`) are supplied privately in Dokploy. @@ -33,7 +40,7 @@ state; enabling a worker requires reviewing the existing scheduled tasks first. Validate changes with: ```sh -python -m unittest discover -s deploy/tests -v +DJANGO_SETTINGS_MODULE=deploy.settings python -m unittest discover -s deploy/tests -v docker compose -f deploy/compose.yml config --quiet ``` diff --git a/deploy/compose.yml b/deploy/compose.yml index d44aa25d..560f2376 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -5,7 +5,7 @@ services: dockerfile: deploy/Dockerfile target: app # Preserve the existing host IPv6 routing used by network tests. - # Gunicorn binds only to loopback; Apache provides public HTTPS. + # Gunicorn serves a private socket; Traefik provides public HTTPS. network_mode: host restart: unless-stopped init: true @@ -31,8 +31,9 @@ services: target: /app/files bind: create_host_path: false + - runtime:/run/testingplatform healthcheck: - test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:18081/', headers={'Host':'testing.nc3.lu'}), timeout=10)"] + test: ["CMD", "python", "-c", "import socket; s=socket.socket(socket.AF_UNIX); s.settimeout(10); s.connect('/run/testingplatform/gunicorn.sock'); s.sendall(b'GET / HTTP/1.0\\r\\nHost: testing.nc3.lu\\r\\n\\r\\n'); assert b' 200 ' in s.recv(64)"] interval: 30s timeout: 15s retries: 3 @@ -47,13 +48,18 @@ services: context: .. dockerfile: deploy/Dockerfile target: proxy - network_mode: host + networks: + - dokploy-network + ports: + - "127.0.0.1:18080:80" + volumes: + - runtime:/run/testingplatform:ro restart: unless-stopped depends_on: web: condition: service_healthy healthcheck: - test: ["CMD", "wget", "-q", "--spider", "--header=Host: testing.nc3.lu", "http://127.0.0.1:18080/"] + test: ["CMD", "wget", "-q", "--spider", "--header=Host: testing.nc3.lu", "http://127.0.0.1/"] interval: 30s timeout: 15s retries: 3 @@ -62,3 +68,10 @@ services: options: max-size: 10m max-file: "3" + +networks: + dokploy-network: + external: true + +volumes: + runtime: diff --git a/deploy/nginx.conf b/deploy/nginx.conf index 2771b01e..8f9eb961 100644 --- a/deploy/nginx.conf +++ b/deploy/nginx.conf @@ -2,8 +2,11 @@ map $http_x_forwarded_proto $original_proto { default $scheme; https https; } +upstream testingplatform_django { + server unix:/run/testingplatform/gunicorn.sock; +} server { - listen 127.0.0.1:18080; + listen 80; server_name testing.nc3.lu; client_max_body_size 100m; location /static/ { @@ -12,7 +15,7 @@ server { expires 7d; } location / { - proxy_pass http://127.0.0.1:18081; + proxy_pass http://testingplatform_django; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/deploy/settings.py b/deploy/settings.py index e460907b..a45f7e5a 100644 --- a/deploy/settings.py +++ b/deploy/settings.py @@ -3,6 +3,10 @@ from testing_platform.settings import * # noqa: F403 from testing_platform.settings import BASE_DIR, DATABASES +# Traefik sets the scheme at the public edge; nginx preserves it on the private +# socket. The application has no public HTTP listener. +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + # SQLite must open an existing database, never silently create an empty one. DATABASES = { "default": { diff --git a/deploy/start.sh b/deploy/start.sh index c5200f5b..a012cec8 100644 --- a/deploy/start.sh +++ b/deploy/start.sh @@ -6,5 +6,6 @@ if [ ! -s db/db.sqlite3 ]; then exit 78 fi exec python -m gunicorn testing_platform.wsgi:application \ - --bind 127.0.0.1:18081 --workers 1 --worker-class gthread --threads 2 \ + --bind unix:/run/testingplatform/gunicorn.sock --umask 0111 \ + --workers 1 --worker-class gthread --threads 2 \ --timeout 330 --graceful-timeout 330 --access-logfile - --error-logfile - diff --git a/deploy/tests/test_proxy_scheme.py b/deploy/tests/test_proxy_scheme.py new file mode 100644 index 00000000..a1e6c652 --- /dev/null +++ b/deploy/tests/test_proxy_scheme.py @@ -0,0 +1,33 @@ +import unittest + +import django +from django.test import RequestFactory, override_settings + +from deploy import settings as deployment_settings + +django.setup() + + +class ProxySchemeTests(unittest.TestCase): + def setUp(self): + self.enterContext( + override_settings( + ALLOWED_HOSTS=["testing.nc3.lu"], + SECURE_PROXY_SSL_HEADER=getattr( + deployment_settings, "SECURE_PROXY_SSL_HEADER", None + ), + ) + ) + + def test_public_https_generates_https_links_through_the_internal_http_proxy(self): + request = RequestFactory().get( + "/login/", + HTTP_HOST="testing.nc3.lu", + HTTP_X_FORWARDED_PROTO="https", + ) + self.assertTrue(request.is_secure()) + self.assertEqual(request.build_absolute_uri(), "https://testing.nc3.lu/login/") + + def test_plain_internal_http_is_not_treated_as_https(self): + request = RequestFactory().get("/", HTTP_HOST="testing.nc3.lu") + self.assertFalse(request.is_secure()) diff --git a/deploy/tests/test_start_guard.py b/deploy/tests/test_start_guard.py index 97c5860b..87e23189 100644 --- a/deploy/tests/test_start_guard.py +++ b/deploy/tests/test_start_guard.py @@ -42,6 +42,7 @@ def test_present_database_is_retained_and_start_does_not_run_migrations(self): ) self.assertEqual(r.returncode, 0, r.stderr) self.assertIn("gunicorn", r.stdout) + self.assertIn("unix:/run/testingplatform/gunicorn.sock", r.stdout) self.assertNotIn("migrate", r.stdout) self.assertEqual(db.read_bytes(), before)