product-scraper

.github/workflows/docker-publish.yml

@@ -6,28 +6,25 @@ on:
 
 jobs:
   lint:
-    name: Pylint
     runs-on: ubuntu-latest
     permissions:
       contents: read
+    strategy:
+      matrix:
+        python-version: ["3.12"]
     steps:
-      - name: Check out the repo
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-          cache: pip
-
-      - name: Install dependencies
-        run: pip install pylint
-
-      - name: Run pylint
-        run: pylint app/ main.py --output-format=colorized --exit-zero
-        continue-on-error: true
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pylint && pip install -r requirements.txt
+    - name: Analyse the code with pylint
+      run: |
+        pylint --rcfile=.pylintrc $(git ls-files '*.py' ':!migrations')
 
   push_to_registry:
     name: Push Docker image to Docker Hub

.github/workflows/pylint.yml

@@ -0,0 +1,25 @@
+name: Pylint
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pylint && pip install -r requirements.txt
+    - name: Analyse the code with pylint
+      run: |
+        pylint --rcfile=.pylintrc $(git ls-files '*.py' ':!migrations')

CLAUDE.md

@@ -69,13 +69,13 @@ AppSetting (key/value table for all runtime config: scheduler, notifications, en
 
 **Shop adapters** (`app/shop_adapters/`): Subclass `BaseAdapter`, set `domain` and optionally `fetch_engine = "cloudscraper"`, implement `extract(html, url) → AdapterResult`. Register in `registry.py` via `_reg(YourAdapter())`. Currently registered: `3djake.de`, `prusa3d.com`, `anycubic.com`, `eu.store.bambulab.com`, `esun3dstore.com`, `esun3dstoreeu.com`, `elegoo.com`.
 
-**Price parsing:** `parse_price()` in `routes/shop_rules.py` handles German (`1.299,00 €`) and English (`1,299.00`) formats, plus JSON-LD key fragments. Import from there when needed elsewhere.
+**Price parsing:** `parse_price()` in `routes/shop_rules.py` handles Currency formats, plus JSON-LD key fragments. Import from there when needed elsewhere.
 
 **Notification settings:** Stored in `AppSetting` key/value rows. Access via `app/settings_service.py` (`get_all` / `set_many`). Send via `app/notification_service.py`. Keys: `discord.enabled`, `discord.webhook_url`, `smtp.*`.
 
 ## Key conventions
 
-**Language:** All code and user-facing text must be English — no Other Language words anywhere in this repo (open-source on GitHub). This includes flash messages, validation errors, log messages, notification/email/Discord copy, comments, and seed data. Conversation with the user may stay in German per their global preference, but nothing written to a file in this repo may contain German. Double-check every new or edited string before finishing a task — this has been missed before (e.g. `app/notification_service.py`, `app/alert_service.py` initially shipped with German copy and had to be fixed after the fact).
+**Language:** All code and user-facing text must be English — no Other Language words anywhere in this repo (open-source on GitHub). This includes flash messages, validation errors, log messages, notification/email/Discord copy, comments, and seed data.
 
 **CSRF:** Every state-changing form needs `<input type="hidden" name="_csrf_token" value="{{ csrf_token() }}">`. The `before_request` hook validates it on POST/PUT/PATCH/DELETE. JSON requests (content-type contains `json`) are exempt.
 

README.md

@@ -219,7 +219,7 @@ To add a new adapter: subclass `BaseAdapter` in `app/shop_adapters/`, implement
 
 ### ShopRules (generic)
 
-For any other shop: create a rule at `/shop-rules` with domain, CSS price selector, and optional regex — or use the visual point-and-click picker to build one without touching CSS. German (`1.299,00 €`) and English (`1,299.00`) price formats are detected automatically.
+For any other shop: create a rule at `/shop-rules` with domain, CSS price selector, and optional regex — or use the visual point-and-click picker to build one without touching CSS.
 
 ---
 

app/price_check_service.py

@@ -28,12 +28,17 @@
 _HTTPX_TIMEOUT = 12.0
 
 
-async def _fetch_httpx(url: str) -> tuple[str | None, str | None]:
+async def _fetch_httpx(
+    url: str, extra_headers: dict | None = None, warmup_url: str | None = None
+) -> tuple[str | None, str | None]:
     import httpx
+    headers = {**_HTTPX_HEADERS, **extra_headers} if extra_headers else _HTTPX_HEADERS
     try:
         async with httpx.AsyncClient(
-            follow_redirects=True, timeout=_HTTPX_TIMEOUT, headers=_HTTPX_HEADERS
+            follow_redirects=True, timeout=_HTTPX_TIMEOUT, headers=headers
         ) as client:
+            if warmup_url:
+                await client.get(warmup_url)
             resp = await client.get(url)
             resp.raise_for_status()
             return resp.text, None
@@ -124,7 +129,11 @@ async def check_price(
     elif effective_engine == "playwright":
         html, fetch_error = await _fetch_playwright(link_url, settings)
     else:
-        html, fetch_error = await _fetch_httpx(link_url)
+        html, fetch_error = await _fetch_httpx(
+            link_url,
+            adapter.fetch_headers(link_url) if adapter else None,
+            adapter.warmup_url(link_url) if adapter else None,
+        )
 
     if fetch_error:
         await _save_error(link_id, link_currency, fetch_error)

app/routes/api_keys.py

@@ -15,8 +15,8 @@
 def admin_required(f):
     @wraps(f)
     async def wrapper(*args, **kwargs):
-        async with get_db() as session:
-            user = await session.get(User, int(current_user.auth_id))
+        async with get_db() as db:
+            user = await db.get(User, int(current_user.auth_id))
         if not user or user.role != UserRole.admin:
             abort(403)
         return await f(*args, **kwargs)
@@ -67,11 +67,11 @@ async def create():
 @login_required
 @admin_required
 async def delete(key_id: int):
-    async with get_db() as session:
-        key = await session.get(ApiKey, key_id)
+    async with get_db() as db:
+        key = await db.get(ApiKey, key_id)
         if not key:
             abort(404)
-        await session.delete(key)
+        await db.delete(key)
 
     await flash("API key revoked.", "success")
     return redirect(url_for("api_keys.index"))

app/routes/shop_rules.py

@@ -41,7 +41,7 @@ async def wrapper(*args, **kwargs):
 
 def parse_price(raw: str) -> float:
     """
-    Parse price strings in German or English format to float.
+    Parse price strings format to float.
       "17,99 €"              → 17.99
       "€17.99"               → 17.99
       "1.299,00 €"           → 1299.00
@@ -61,10 +61,8 @@ def parse_price(raw: str) -> float:
     comma_pos = s.rfind(",")
 
     if comma_pos > dot_pos:
-        # German: 1.299,00 — last separator is comma → decimal
         s = s.replace(".", "").replace(",", ".")
     elif dot_pos > comma_pos:
-        # English or plain float: 1,299.00 or 32.990000 — last separator is dot
         s = s.replace(",", "")
     else:
         s = s.replace(",", ".")

app/shop_adapters/_3djake.py

@@ -8,13 +8,21 @@
 
 class ThreeDJakeAdapter(BaseAdapter):
     domains = ("3djake.de",)
+    fetch_engine = "httpx"
 
     _PRICE_SEL = ".price"
     _PRICE_RE  = r"\d+[,\.]\d{2}"
     _TITLE_SEL = "h1"
     _AVAIL_SEL = "[class*='availab']"
 
     def extract(self, html: str, url: str) -> AdapterResult:
+        title_node = _extract(html, "title", None)
+        if title_node and "Request Problem" in title_node:
+            return AdapterResult(
+                status="blocked",
+                error_message="niceshops WAF blocked the request (Error 425 - Security Filtering).",
+            )
+
         title        = _extract(html, self._TITLE_SEL, None)
         price_raw    = _extract(html, self._PRICE_SEL, self._PRICE_RE)
         availability = _extract(html, self._AVAIL_SEL, None)

app/shop_adapters/_amazon.py

@@ -0,0 +1,89 @@
+"""
+Amazon adapter — all major Amazon marketplaces, confirmed working via plain httpx.
+"""
+from urllib.parse import urlparse
+
+from selectolax.parser import HTMLParser
+
+from app.routes.shop_rules import parse_price
+from .base import BaseAdapter, AdapterResult
+
+_ACCEPT_LANGUAGE = {
+    "amazon.com": "en-US,en;q=0.9",
+    "amazon.de": "de-DE,de;q=0.9,en;q=0.8",
+    "amazon.co.uk": "en-GB,en;q=0.9",
+    "amazon.fr": "fr-FR,fr;q=0.9,en;q=0.8",
+    "amazon.it": "it-IT,it;q=0.9,en;q=0.8",
+    "amazon.es": "es-ES,es;q=0.9,en;q=0.8",
+    "amazon.nl": "nl-NL,nl;q=0.9,en;q=0.8",
+    "amazon.se": "sv-SE,sv;q=0.9,en;q=0.8",
+    "amazon.pl": "pl-PL,pl;q=0.9,en;q=0.8",
+    "amazon.co.jp": "ja-JP,ja;q=0.9,en;q=0.8",
+    "amazon.ca": "en-CA,en;q=0.9",
+    "amazon.com.au": "en-AU,en;q=0.9",
+    "amazon.in": "en-IN,en;q=0.9",
+}
+
+
+class AmazonAdapter(BaseAdapter):
+    domains = tuple(_ACCEPT_LANGUAGE)
+    fetch_engine = "httpx"
+
+    def fetch_headers(self, url: str) -> dict | None:
+        domain = (urlparse(url).hostname or "").removeprefix("www.")
+        lang = _ACCEPT_LANGUAGE.get(domain)
+        return {"Accept-Language": lang} if lang else None
+
+    def extract(self, html: str, url: str) -> AdapterResult:
+        tree = HTMLParser(html)
+
+        is_captcha = (
+            tree.css_first("form[action*='validateCaptcha']") is not None
+            or "/errors/validateCaptcha" in url
+            or "Enter the characters you see below" in html
+            or "Type the characters you see in this image" in html
+        )
+        if is_captcha:
+            return AdapterResult(
+                status="blocked",
+                error_message="Amazon returned a CAPTCHA challenge page.",
+            )
+
+        title_node = tree.css_first("#productTitle")
+        title = title_node.text(strip=True) if title_node else None
+
+        price_node = (
+                tree.css_first(".a-price .a-offscreen")
+                or tree.css_first("#priceblock_ourprice")
+                or tree.css_first("#priceblock_dealprice")
+        )
+        price_raw = price_node.text(strip=True) if price_node else None
+
+        if not price_raw:
+            return AdapterResult(
+                status="error",
+                error_message="Price element not found — page structure may have changed or product is unavailable.",
+                title=title,
+            )
+
+        try:
+            price_parsed = parse_price(price_raw)
+        except ValueError as e:
+            return AdapterResult(
+                status="error",
+                price_raw=price_raw,
+                error_message=f"Price parse failed: {price_raw!r} → {e}",
+                title=title,
+            )
+
+        avail_node = tree.css_first("#availability .primary-availability-message") \
+                     or tree.css_first("#availability span")
+        availability = avail_node.text(strip=True) if avail_node else None
+
+        return AdapterResult(
+            status="success",
+            price_raw=price_raw,
+            price_parsed=price_parsed,
+            availability=availability,
+            title=title,
+        )

app/shop_adapters/_anycubic.py

@@ -18,6 +18,7 @@
 
 class AnycubicAdapter(BaseAdapter):
     domains = ("anycubic.com",)
+    fetch_engine = "httpx"
 
     def extract(self, html: str, url: str) -> AdapterResult:
         title = _extract(html, "h1", None)