Django 5 upgrade

.github/workflows/lambda.yml

@@ -1,26 +1,37 @@
 name: Post-Deploy Lambda
-
 on:
   deployment_status:
-
 jobs:
   deploy-lambdas:
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-1
     steps:
       - uses: actions/checkout@v3
-
+        with:
+          lfs: true
+      - name: Install Git LFS and pull files
+        run: |
+          sudo apt-get install -y git-lfs
+          git lfs install
+          git lfs fetch --all
+          git lfs checkout
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
       - name: Show deployment info
         run: |
           echo "Deployment environment: ${{ github.event.deployment.environment }}"
-
       - name: Run Lambda production deploy
         if: >
           github.event.deployment.environment == 'documentcloud-prod' &&
           github.event.deployment_status.state == 'success'
         run: |
           echo "Deploying production lambda updates"
           bash config/aws/lambda/codeship_deploy_lambdas.sh prod-lambda
-
       - name: Run Lambda staging deploy
         if: >
           github.event.deployment.environment == 'documentcloud-staging' &&

.github/workflows/test.yml

@@ -1,15 +1,13 @@
 name: CI
-
 on:
   workflow_dispatch:
   pull_request:
   push:
     branches:
       - master
       - staging
-
 env:
-  PYENV_VERSION: '3.10'
+  PYENV_VERSION: '3.12'
   DJANGO_SECRET_KEY: ${{secrets.SECRET_KEY}}
   DJANGO_SETTINGS_MODULE: config.settings.test
   DATABASE_URL: postgres://${PGUSER}:${PGPASSWORD}@127.0.0.1:5432/test
@@ -38,15 +36,10 @@ env:
   DOCUMENT_BUCKET: ${{secrets.DOCUMENT_BUCKET}}
   REDIS_PROCESSING_PASSWORD: ""
   SAM_CLI_TELEMETRY: 0
-
-
-
 
-
 jobs:
   check:
     runs-on: ubuntu-latest
-
     services:
       redis:
         image: redis
@@ -63,30 +56,19 @@ jobs:
             submodules: recursive
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
           cache: "pip" # caching pip dependencies
-
-      - name: Install pip
-        run: |
-          wget https://bootstrap.pypa.io/pip/3.6/get-pip.py
-          python3 get-pip.py
-
       - name: Install
         run: |
           pip install -r requirements/local.txt
-
       - name: Lint
         run: pylint documentcloud
-
       - name: Isort
-        run: isort --check-only -rc documentcloud
-
+        run: isort --check-only --diff documentcloud
       - name: Formatting
         run: black --check documentcloud --exclude migrations
-
   test:
     runs-on: ubuntu-latest
-
     services:
       redis:
         image: redis
@@ -97,7 +79,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-
       postgres:
         image: postgres
         env:
@@ -113,33 +94,21 @@ jobs:
         ports:
           # Maps tcp port 5432 on service container to the host
           - 5432:5432
-
     steps:
-
       - name: Checkout
         uses: actions/checkout@v4
-
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
           cache: "pip" # caching pip dependencies
-
-      - name: Install pip
-        run: |
-          wget https://bootstrap.pypa.io/pip/3.6/get-pip.py
-          python3 get-pip.py
-
       - name: Install requirements
         run: |
           pip install -r requirements/local.txt
-
-
       - name: Test
         run:  pytest documentcloud -m "not solr"
         env:
           # use the credentials for the service container
           PG_USER: test
           PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
           DATABASE_URL: postgres://test:postgres@127.0.0.1:5432/test
-

.gitignore

@@ -306,3 +306,4 @@ CLAUDE.md
 .claude
 
 rootCA.pem
+tests.txt

.pylintrc

@@ -1,7 +1,6 @@
 [MASTER]
-# https://stackoverflow.com/a/39207275/10952222
-init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.join(os.path.dirname(find_pylintrc()), 'documentcloud'))"
-load-plugins=pylint_django, pylint_celery
+init-hook="import sys, os; sys.path.insert(0, '/app'); os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')"
+load-plugins=pylint_django
 ignore=migrations
 
 [FORMAT]
@@ -11,7 +10,7 @@ max-args=6
 
 [MESSAGES CONTROL]
 enable=useless-suppression
-disable=missing-docstring,too-many-ancestors,too-few-public-methods,no-else-return,no-member,attribute-defined-outside-init,similarities,import-outside-toplevel,cyclic-import,raise-missing-from,django-not-configured
+disable=missing-docstring,too-many-ancestors,too-few-public-methods,no-else-return,no-member,attribute-defined-outside-init,similarities,import-outside-toplevel,cyclic-import,raise-missing-from,django-not-configured,wrong-import-order,wrong-import-position
 
 [TYPECHECK]
 generated-members=REQUEST,acl_users,aq_parent,"[a-zA-Z]+_set{1,2}",save,delete

README.md

@@ -56,10 +56,7 @@ You must first have these set up and ready to go:
       ```
 12. Go to [Django admin for DocumentCloud](https://api.dev.documentcloud.org/admin) and add the required static [flat page](https://api.dev.documentcloud.org/admin/flatpages/flatpage/) called `/tipofday/`. It can be blank. Do not prefix the URL with `/pages/`. Specifying the `Site` as `example.com` is alright.
 13. Create an initial Minio bucket to simulate AWS S3 locally:
-    - Reference your DocumentCloud `.django` file for these variables:
-    - Visit the `MINIO_URL` with a browser, likely at [this address](http://minio.documentcloud.org:9000), and login with the minio `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY`
-    - At the bottom right corner click the round plus button and then click the first circle that appears above it to "create bucket".
-    - Create a bucket called `documents`
+    - Run `inv initialize-minio`
 14. Upload a document:
     - **Check your memory allocation on Docker is at least 7gb.** A sign that you do not have enough memory allocated is if containers are randomly failing or if your system is swapping heavily, especially when uploading documents.
     - The "upload" button should not be grayed out (if it is, check your user organization Verified Journalist status above)

compose/local/django/Dockerfile

@@ -1,33 +1,27 @@
-# FROM python:3.6-alpine
-FROM matthewfeickert/docker-python3-ubuntu:3.10.5
-
+FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
-
 USER root
 
 RUN apt-get -qq -y update && \
   apt-get -qq -y install \
-  # Pip dependencies
-  python3-pip \
+  # Build dependencies
+  build-essential \
   # Postgres dependencies
   libpq-dev postgresql-client python3-psycopg2 \
   # Tesseract dependencies
-  libjpeg-turbo8 libtiff5 \
+  libjpeg62-turbo libtiff6 \
   # LibreOffice dependencies
-  libnss3-dev libcurl4-nss-dev libxslt1-dev libpixman-1-0 libxcb-render0-dev && \
-  # Symlink bash and python
-  ln -sf bash /bin/sh && rm -f /usr/bin/python && \
-  ln -s /usr/bin/python3 /usr/bin/python && \
-  curl https://bootstrap.pypa.io/pip/3.6/get-pip.py -o get-pip.py && \
-  python3 get-pip.py --force-reinstall
+  libnss3-dev libcurl4-openssl-dev libxslt1-dev libpixman-1-0 libxcb-render0-dev \
+  # Utilities
+  curl git
 
 # set up makecert root CA
 RUN curl http://localhost/rootCA.pem > /usr/local/share/ca-certificates/rootCA.crt && update-ca-certificates
 
 # Requirements are installed here to ensure they will be cached.
 COPY ./requirements /requirements
-# RUN pip install --upgrade pip && pip install -r /requirements/local.txt
-RUN pip install -r /requirements/local.txt
+
+RUN pip install --upgrade "pip>=26,<27" && pip install -r /requirements/local.txt
 
 COPY ./compose/production/django/entrypoint /entrypoint
 RUN sed -i 's/\r//' /entrypoint && chmod +x /entrypoint
@@ -48,11 +42,8 @@ RUN sed -i 's/\r//' /start-flower && chmod +x /start-flower
 # Entry point
 # =-=-=-=-=-=
 
-# Temporary measure to get pip-compile to work
-# RUN pip install 'pip<19.2'
-
 WORKDIR /app
 
 ENV LD_LIBRARY_PATH /app/documentcloud/documents/processing/ocr/tesseract
 
-ENTRYPOINT ["/entrypoint"]
+ENTRYPOINT ["/entrypoint"]

compose/production/django/Dockerfile

@@ -1,23 +1,20 @@
-
-FROM python:3.6-alpine
-
+FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
 
-RUN apk update \
+RUN apt-get -qq -y update && apt-get -qq -y install \
   # psycopg2 dependencies
-  && apk add --virtual build-deps gcc python3-dev musl-dev \
-  && apk add postgresql-dev \
+  gcc python3-dev libpq-dev \
   # Pillow dependencies
-  && apk add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \
+  libjpeg62-turbo zlib1g-dev libfreetype6-dev liblcms2-dev libopenjp2-7-dev libtiff6 tk-dev tcl-dev \
   # CFFI dependencies
-  && apk add libffi-dev py-cffi
+  libffi-dev python3-cffi
 
-RUN addgroup -S django \
-    && adduser -S -G django django
+RUN groupadd -r django \
+    && useradd -r -g django django
 
-# Requirements are installed here to ensure they will be cached.
+# Requirements are installed here to ensure they will be cached. 
 COPY ./requirements /requirements
-RUN pip install --no-cache-dir -r /requirements/production.txt \
+RUN pip install --upgrade "pip>=26,<27" && pip install --no-cache-dir -r /requirements/production.txt \
     && rm -rf /requirements
 
 COPY ./compose/production/django/entrypoint /entrypoint

config/aws/lambda/cloud-requirements.txt

@@ -1,2 +1,3 @@
-boto3==1.10.14
-smart-open==1.8.4
+boto3==1.40.61
+smart-open==7.5.1
+setuptools<81

config/aws/lambda/info-and-image-requirements.txt

@@ -1,4 +1,5 @@
-boto3==1.21.21
-smart-open==1.8.4
-pdfplumber==0.5.28
-pikepdf==5.4.0
+boto3==1.40.61
+smart-open==7.5.1
+pdfplumber==0.11.9
+pikepdf==10.5.1
+setuptools<81