fix: CPU bench

.github/workflows/cpu-benchmark.yml

@@ -0,0 +1,38 @@
+name: Python SDK py-spy CPU benchmark
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - labeled
+
+permissions:
+  contents: read
+
+jobs:
+  # Timing and profiling run as parallel jobs on separate runners so py-spy
+  # sampling never contends with the timed run.
+  cpu-timing:
+    name: CPU benchmark timing
+    # Needs to match the arch the baseline was generated on.
+    runs-on: ubuntu-24.04-arm
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'check-cpu-benchmark') &&
+      (
+        github.event.pull_request.author_association == 'COLLABORATOR' ||
+        github.event.pull_request.author_association == 'MEMBER' ||
+        github.event.pull_request.author_association == 'OWNER'
+      )
+    steps:
+      - uses: actions/checkout@v4
+
+      # Build the perf image (shared with the memory benchmark).
+      - name: Build perf image
+        run: make perf-image-rebuild
+
+      # Uses the Dockerfile environment for repeatable runs. Report-only:
+      # baseline deltas land in this job's step summary, exit code stays 0.
+      - name: Run CPU timing benchmark
+        run: make cpu-bench CPU_MODE=timing

.github/workflows/memory-benchmark.yml

@@ -40,5 +40,5 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: memray-flamegraphs
-          path: tests/perf/reports/*.html
+          path: tests/perf/memory/reports/*.html
           if-no-files-found: warn

.gitignore

@@ -124,5 +124,7 @@ src/c2pa/libs/
 !tests/fixtures/*.key
 
 # Memory profiling reports
-tests/perf/reports/*.html
-tests/perf/reports/*.bin
+tests/perf/memory/reports/*.html
+tests/perf/memory/reports/*.bin
+tests/perf/cpu/reports/*.svg
+tests/perf/cpu/reports/*.speedscope.json

Makefile

@@ -62,10 +62,6 @@ test:
 	$(PYTHON) ./tests/test_unit_tests.py
 	$(PYTHON) ./tests/test_unit_tests_threaded.py
 
-# Runs benchmarks in the venv
-benchmark:
-	$(PYTHON) -m pytest tests/benchmark.py -v
-
 # Tests building and installing a local wheel package
 # Downloads required artifacts, builds the wheel, installs it, and verifies the installation
 test-local-wheel-build:
@@ -141,11 +137,16 @@ build-from-source:
 docs:
 	$(PYTHON) scripts/generate_api_docs.py
 
-# Memory profiling with memray (runs in Docker, reports go to tests/perf/reports/)
-# More details for usage are in tests/perf/README.md
+# Performance benchmarks (run in Docker):
+# - memory (memray): reports go to tests/perf/memory/reports/, docs in tests/perf/memory/README.md
+# - cpu (py-spy): reports go to tests/perf/cpu/reports/, docs in tests/perf/cpu/README.md
 PERF_ENV ?= python-3.12-slim
 MEMRAY_ITERATIONS ?= 100
 MEMRAY_THRESHOLD ?= 1.1
+CPU_ITERATIONS ?= 100
+CPU_MODE ?= all
+CPU_REPEATS ?= 0
+PYSPY_RATE ?= 100
 SCENARIO ?=
 SCENARIO_ARG := $(if $(SCENARIO),--scenario $(SCENARIO),)
 # In CI, use en vars to write the report to the job run
@@ -156,23 +157,36 @@ GH_SUMMARY_MOUNT := $(if $(GITHUB_STEP_SUMMARY),-v $(GITHUB_STEP_SUMMARY):$(GITH
 # change (use perf-image-rebuild for that).
 .PHONY: perf-image
 perf-image:
-	@docker image inspect c2pa-memray-$(PERF_ENV) >/dev/null 2>&1 || \
-		docker build -f tests/perf/Dockerfiles/$(PERF_ENV)-perf-Dockerfile -t c2pa-memray-$(PERF_ENV) .
+	@docker image inspect c2pa-perf-$(PERF_ENV) >/dev/null 2>&1 || \
+		docker build -f tests/perf/Dockerfiles/$(PERF_ENV)-perf-Dockerfile -t c2pa-perf-$(PERF_ENV) .
 
-# Force a clean rebuild of the memray perf Docker image
+# Force a clean rebuild of the perf Docker image (shared by memory and cpu benchmarks)
 .PHONY: perf-image-rebuild
 perf-image-rebuild:
-	docker build --no-cache --pull -f tests/perf/Dockerfiles/$(PERF_ENV)-perf-Dockerfile -t c2pa-memray-$(PERF_ENV) .
+	docker build --no-cache --pull -f tests/perf/Dockerfiles/$(PERF_ENV)-perf-Dockerfile -t c2pa-perf-$(PERF_ENV) .
 
 # Runs memory benchmarks. Pre-requisite: Docker image built using `make perf-image-rebuild`.
 .PHONY: memory-use-bench
 memory-use-bench:
-	docker run --rm -v $(PWD):/workspace $(GH_SUMMARY_MOUNT) -e PYTHONPATH=/workspace/src -e PERF_ENV=$(PERF_ENV) -e MEMRAY_ITERATIONS=$(MEMRAY_ITERATIONS) -e MEMRAY_THRESHOLD=$(MEMRAY_THRESHOLD) -e GITHUB_TOKEN -e GITHUB_STEP_SUMMARY c2pa-memray-$(PERF_ENV) python -m tests.perf.run_profile $(SCENARIO_ARG) $(PERF_ARGS)
+	docker run --rm -v $(PWD):/workspace $(GH_SUMMARY_MOUNT) -e PYTHONPATH=/workspace/src -e PERF_ENV=$(PERF_ENV) -e MEMRAY_ITERATIONS=$(MEMRAY_ITERATIONS) -e MEMRAY_THRESHOLD=$(MEMRAY_THRESHOLD) -e GITHUB_TOKEN -e GITHUB_STEP_SUMMARY c2pa-perf-$(PERF_ENV) python -m tests.perf.memory.run_profile $(SCENARIO_ARG) $(PERF_ARGS)
 	@echo ""
-	@echo "Reports written to tests/perf/reports/"
-	@echo "Open tests/perf/reports/<scenario>-{peak,leaks,temporary}.html in a browser"
+	@echo "Reports written to tests/perf/memory/reports/"
+	@echo "Open tests/perf/memory/reports/<scenario>-{peak,leaks,temporary}.html in a browser"
 
 .PHONY: clean-memory-perf-reports
 clean-memory-perf-reports:
-	rm -f tests/perf/reports/*.html tests/perf/reports/*.bin
-	@echo "Cleared tests/perf/reports/"
+	rm -f tests/perf/memory/reports/*.html tests/perf/memory/reports/*.bin
+	@echo "Cleared tests/perf/memory/reports/"
+
+# Runs CPU benchmarks (timing metrics+py-spy flamegraphs).
+# Pre-requisite: Docker image built using `make perf-image-rebuild`.
+.PHONY: cpu-bench
+cpu-bench:
+	docker run --rm --cap-add SYS_PTRACE --security-opt seccomp=unconfined -v $(PWD):/workspace $(GH_SUMMARY_MOUNT) -e PYTHONPATH=/workspace/src -e PERF_ENV=$(PERF_ENV) -e CPU_ITERATIONS=$(CPU_ITERATIONS) -e CPU_REPEATS=$(CPU_REPEATS) -e PERF_DISABLE_TSA -e PYSPY_RATE=$(PYSPY_RATE) -e PYSPY_FORMAT -e GITHUB_TOKEN -e GITHUB_STEP_SUMMARY c2pa-perf-$(PERF_ENV) python -m tests.perf.cpu.run_profile --mode $(CPU_MODE) $(SCENARIO_ARG) $(PERF_ARGS)
+	@echo ""
+	@echo "Reports written to tests/perf/cpu/reports/"
+
+.PHONY: clean-cpu-perf-reports
+clean-cpu-perf-reports:
+	rm -f tests/perf/cpu/reports/*.svg tests/perf/cpu/reports/*.speedscope.json
+	@echo "Cleared tests/perf/cpu/reports/"

requirements-dev.txt

@@ -6,7 +6,6 @@ toml==0.10.2  # For reading pyproject.toml files
 
 # Testing dependencies
 pytest>=8.1.0
-pytest-benchmark>=5.1.0
 
 # for downloading the library artifacts
 requests>=2.0.0

tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile

@@ -13,10 +13,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY requirements.txt requirements-dev.txt ./
 RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt
 
-RUN pip install --no-cache-dir memray==1.19.3
+RUN pip install --no-cache-dir memray==1.19.3 py-spy==0.4.2
 
 COPY tests/perf/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
 ENTRYPOINT ["/entrypoint.sh"]
-CMD ["python", "-m", "tests.perf.run_profile"]
+CMD ["python", "-m", "tests.perf.memory.run_profile"]

tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile

@@ -13,10 +13,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY requirements.txt requirements-dev.txt ./
 RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt
 
-RUN pip install --no-cache-dir memray==1.19.3
+RUN pip install --no-cache-dir memray==1.19.3 py-spy==0.4.2
 
 COPY tests/perf/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
 ENTRYPOINT ["/entrypoint.sh"]
-CMD ["python", "-m", "tests.perf.run_profile"]
+CMD ["python", "-m", "tests.perf.memory.run_profile"]

tests/perf/Dockerfiles/ubuntu-22.04-perf-Dockerfile

@@ -22,10 +22,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY requirements.txt ./
 RUN pip3 install --no-cache-dir -r requirements.txt
 
-RUN pip3 install --no-cache-dir memray==1.19.3 requests==2.34.2
+RUN pip3 install --no-cache-dir memray==1.19.3 py-spy==0.4.2 requests==2.34.2
 
 COPY tests/perf/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
 ENTRYPOINT ["/entrypoint.sh"]
-CMD ["python", "-m", "tests.perf.run_profile"]
+CMD ["python", "-m", "tests.perf.memory.run_profile"]

tests/perf/Dockerfiles/ubuntu-24.04-perf-Dockerfile

@@ -22,10 +22,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY requirements.txt ./
 RUN pip3 install --no-cache-dir --break-system-packages -r requirements.txt
 
-RUN pip3 install --no-cache-dir --break-system-packages memray==1.19.3 requests==2.34.2
+RUN pip3 install --no-cache-dir --break-system-packages memray==1.19.3 py-spy==0.4.2 requests==2.34.2
 
 COPY tests/perf/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
 ENTRYPOINT ["/entrypoint.sh"]
-CMD ["python", "-m", "tests.perf.run_profile"]
+CMD ["python", "-m", "tests.perf.memory.run_profile"]