Fix search crash, stale keyword flag and TED/UNGM config issues + devops (release and CI)

.github/workflows/docker.yml

@@ -0,0 +1,49 @@
+name: Docker image
+
+# Builds the image and pushes it to GHCR when a version tag (e.g. 1.0.62) is
+# pushed. Can also be started by hand from the Actions tab.
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  IMAGE: ghcr.io/${{ github.repository }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: docker/setup-buildx-action@v4
+
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Version tag -> ghcr.io/eaudeweb/scratch:1.0.62 and :latest
+      - id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=sha
+
+      - uses: docker/build-push-action@v7
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/tests.yml

@@ -0,0 +1,84 @@
+name: Tests
+
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]
+
+permissions:
+  contents: read
+
+# A new push to the same branch or pull request cancels the previous run
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    # Same major versions as docker-compose.yml
+    services:
+      postgres:
+        image: postgres:15-alpine
+        env:
+          POSTGRES_DB: scratch
+          POSTGRES_USER: scratch
+          POSTGRES_PASSWORD: scratch
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U scratch"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      elasticsearch:
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.17.8
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: "false"
+          ES_JAVA_OPTS: -Xms512m -Xmx512m
+        ports:
+          - 9200:9200
+        options: >-
+          --health-cmd "curl -fs http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 20
+      redis:
+        image: redis:alpine
+        ports:
+          - 6379:6379
+
+    env:
+      POSTGRES_HOST: localhost
+      POSTGRES_DB: scratch
+      POSTGRES_USER: scratch
+      POSTGRES_PASSWORD: scratch
+      ELASTICSEARCH_HOST: localhost:9200
+      REDIS_HOST: localhost
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: requirements*.txt
+
+      # Headers needed to build python-ldap and psycopg2
+      - name: Install system packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends libldap-dev libsasl2-dev libpq-dev
+
+      - name: Install dependencies
+        run: pip install -r requirements-dev.txt
+
+      - name: Check
+        run: python manage.py check
+
+      - name: Run tests
+        run: python manage.py test app/tests --settings=scratch.test_settings

CLAUDE.md

@@ -36,7 +36,7 @@ python manage.py test app.tests.test_ted_parser.TestClassName.test_method --sett
 
 - Test fixtures (sample TED XML, UNGM/IUCN HTML, PDFs/DOCX) live in `app/tests/parser_files/`; model factories are in `app/factories.py` (factory_boy).
 - Tests touching `Keyword` should extend `app.tests.base.BaseTestCase`, which disconnects the Keyword post_save/post_delete signals (otherwise they enqueue a django-q task that re-saves every tender).
-- CI (`.travis.yml`) runs `./manage.py check` then `coverage run --source='.' ./manage.py test app/tests --settings=scratch.test_settings`.
+- CI (`.github/workflows/tests.yml`, on pull requests to and pushes to `master`) runs `./manage.py check` then `./manage.py test app/tests --settings=scratch.test_settings` against Postgres, Elasticsearch and Redis service containers. `.github/workflows/docker.yml` builds and pushes the image to GHCR on version tags; see `Release.md`.
 
 There is no configured linter/formatter.
 

README.md

@@ -1,4 +1,4 @@
-# Scratch ![alt text](https://travis-ci.com/eaudeweb/scratch.svg?branch=master) [![Coverage Status](https://coveralls.io/repos/github/eaudeweb/scratch/badge.svg?branch=master)](https://coveralls.io/github/eaudeweb/scratch?branch=master) [![Docker](https://img.shields.io/docker/cloud/build/eaudeweb/scratch?label=Docker&style=flat)](https://hub.docker.com/r/eaudeweb/scratch/builds)
+# Scratch [![Tests](https://github.com/eaudeweb/scratch/actions/workflows/tests.yml/badge.svg)](https://github.com/eaudeweb/scratch/actions/workflows/tests.yml) [![Docker image](https://github.com/eaudeweb/scratch/actions/workflows/docker.yml/badge.svg)](https://github.com/eaudeweb/scratch/actions/workflows/docker.yml)
 
 ### Python Django Scratch
 
@@ -46,6 +46,7 @@
     python manage.py runserver 0.0.0.0:8000
     ```
 ### [Cachet Setup Info](Cachet.md)
+### [Making a release](Release.md)
 ## Running with NGINX
 
 If you want to use NGINX to serve the project you need to do the following:

Release.md

@@ -0,0 +1,136 @@
+# Releasing Scratch
+
+Releases are Docker images built by GitHub Actions and published to
+`ghcr.io/eaudeweb/scratch`. Production is a single `docker-compose.yml` on the
+server that runs the `web`, `async` and `cron` services from that image (the
+crontab is baked into the image too). Deploying is manual.
+
+```
+git tag 1.0.62 ──> GitHub Actions ──> ghcr.io/eaudeweb/scratch:1.0.62 ──> manual pull on prod
+```
+
+## 1. Before tagging
+
+- Make sure the **Tests** workflow (`.github/workflows/tests.yml`) is green on
+  the pull request. It runs the test suite on every pull request to `master` and
+  every push to `master`, so you don't need to run it yourself.
+- If you changed a scraper, run its update command once locally against a short
+  range (e.g. `python manage.py update_ungm --days_ago 1`). Keep it to one run so
+  the sources don't rate limit or block us.
+- Check for new migrations (`python manage.py makemigrations --check --dry-run`)
+  and note whether the release has any, you'll need it when deploying.
+- Merge the work into `master`, and check that the Tests run on `master` is green
+  before tagging.
+
+## 2. Tag the release
+
+Versions are plain `MAJOR.MINOR.PATCH`, without a `v` prefix. The last image on
+Docker Hub was `1.0.61`, so GHCR releases start at `1.0.62`. To see the latest
+tag, fetch them first (clones don't always have all of them):
+
+```bash
+git fetch --tags
+git tag --sort=-v:refname | head -3
+```
+
+Tag the merge commit on `master` and push the tag:
+
+```bash
+git checkout master && git pull
+git tag 1.0.62
+git push origin 1.0.62
+```
+
+Pushing the tag starts the **Docker image** workflow
+(`.github/workflows/docker.yml`). Follow it in the repository's Actions tab. When
+it is green, the image is available as:
+
+- `ghcr.io/eaudeweb/scratch:1.0.62`
+- `ghcr.io/eaudeweb/scratch:latest`
+- `ghcr.io/eaudeweb/scratch:sha-<commit>`
+
+The workflow can also be started by hand from the Actions tab (*Run workflow*).
+A manual run from a branch only produces the `sha-<commit>` tag, which is useful
+for trying a build without making a release.
+
+## 3. Deploy to production
+
+On the server, in the directory that holds the compose file:
+
+1. Set the new version in the `.env` file next to the compose file:
+
+   ```bash
+   SCRATCH_VERSION=1.0.62
+   ```
+
+   The compose file uses it for the `web`, `async` and `cron` images
+   (`image: ghcr.io/eaudeweb/scratch:${SCRATCH_VERSION}`), so the compose file
+   itself doesn't change between releases. Docker Compose reads `.env` from the
+   project directory automatically. Use a version number rather than `latest`, so
+   it's always clear what is deployed and a rollback is a one-line edit.
+
+2. Pull and restart:
+
+   ```bash
+   docker compose pull
+   docker compose up -d
+   ```
+
+   `pull` downloads the new image while the old containers keep running, so a
+   wrong tag or a registry problem fails before anything is restarted. `up -d`
+   then recreates only the services whose image changed.
+
+3. Check that it came up:
+
+   ```bash
+   docker compose ps
+   docker compose logs --tail=50 web async cron
+   ```
+
+   The Cachet status page should report all components as operational.
+
+### Migrations, static files and the search index
+
+`docker-entrypoint.sh` runs these steps when `web` starts, depending on its
+environment variables (set in the compose file or `docker/app.env`):
+
+| Variable | When `yes`, on every start of `web` |
+|---|---|
+| `DJANGO_MIGRATE` | `manage.py migrate` |
+| `DJANGO_COLLECT_STATIC` | `manage.py collectstatic` |
+| `DJANGO_INDEX_CONTENT` | `manage.py search_index -f --rebuild` (rebuilds every Elasticsearch index, can take a while) |
+
+If migrations are not run automatically and the release has any, run them after
+`up -d`:
+
+```bash
+docker compose exec web python manage.py migrate
+```
+
+## Rolling back
+
+Set `SCRATCH_VERSION` in `.env` back to the previous version and run the same
+`docker compose pull` and `docker compose up -d`. Migrations are not reversed by
+this. If the release added any, check that the old code still works with the new
+schema, or migrate back explicitly
+(`python manage.py migrate app <previous_migration>`) before switching images.
+
+## One-time setup (only when we switched from dockerhub to GitHub packages)
+
+- **Organization settings:** GitHub Actions must be allowed to create packages
+  for the `eaudeweb` organization. The first tag push creates the `scratch`
+  package.
+- **Access from production:** either make the `scratch` package public (the
+  repository already is), or log in once on the server with a personal access
+  token that has the `read:packages` scope:
+
+  ```bash
+  echo <token> | docker login ghcr.io -u <github-user> --password-stdin
+  ```
+
+- **Production compose file:** switch `image:` from `eaudeweb/scratch:...` (Docker
+  Hub) to `ghcr.io/eaudeweb/scratch:${SCRATCH_VERSION}` for `web`, `async` and
+  `cron`.
+
+Images up to `1.0.61` are on Docker Hub as `eaudeweb/scratch`. Docker Hub no
+longer receives new versions.

app/management/commands/add_award.py

@@ -7,12 +7,11 @@
 from time import sleep
 from random import randint
 from app.models import Tender, Award, Vendor
-from app.server_requests import PAYLOAD
+from app.server_requests import PAYLOAD, WINNERS_ENDPOINT_URI
 import logging
 from app.management.commands.base.params import BaseParamsUI
 
 logger = logging.getLogger(__name__)
-WINNERS_ENDPOINT_URI = 'https://www.ungm.org/Public/ContractAward'
 
 CSS_TITLE = 'Title'
 CSS_REFERENCE = 'Reference'

app/management/commands/base/notify.py

@@ -12,8 +12,6 @@
 from app.management.commands.base.params import BaseParamsUI
 from app.utils import emails_to_notify
 
-ENDPOINT_URI = "https://www.ungm.org"
-
 User = get_user_model()
 
 

app/management/commands/update_ungm.py

@@ -7,8 +7,6 @@
 from app.notifications import send_error_email
 from app.utils import log_tenders_update, TenderSource
 
-ENDPOINT_URI = 'https://www.ungm.org'
-
 
 class Command(BaseCommand, BaseParamsUI):
     help = 'Gets all UNGM tenders from the past n days'

app/models.py

@@ -130,7 +130,7 @@ def marked_keyword_title(self):
         if not keywords:
             return title
 
-        regex = r"(" + r"|".join(keywords) + r")"
+        regex = r"(" + r"|".join(map(re.escape, keywords)) + r")"
         return re.sub(regex, r"<mark>\1</mark>", title, flags=re.IGNORECASE)
 
     @cached_property
@@ -140,7 +140,7 @@ def marked_keyword_description(self):
         if not keywords:
             return description
 
-        regex = r"(" + r"|".join(keywords) + r")"
+        regex = r"(" + r"|".join(map(re.escape, keywords)) + r")"
         return re.sub(regex, r"<mark>\1</mark>", description, flags=re.IGNORECASE)
 
     def find_keywords(self, fields):
@@ -161,8 +161,7 @@ def is_favorite_of(self, user):
 
     def save(self, *args, **kwargs):
         keywords = list(self.find_keywords(fields))
-        if keywords:
-            self.has_keywords = True
+        self.has_keywords = bool(keywords)
         super().save(*args, **kwargs)
         self.keywords.set(keywords)