diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
deleted file mode 100644
index dc149bd..0000000
--- a/.github/workflows/ci.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-name: CI
-
-on:
- push:
- branches: ["**"]
- pull_request:
- branches: ["**"]
-
-jobs:
- lint:
- name: Pylint
- runs-on: ubuntu-latest
- permissions:
- contents: read
- steps:
- - name: Check out the repo
- uses: actions/checkout@v4
- with:
- persist-credentials: false
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: "3.12"
- cache: pip
-
- - name: Install dependencies
- run: pip install pylint
-
- - name: Run pylint
- run: pylint app/ main.py --output-format=colorized --exit-zero
- continue-on-error: true
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
index 9eb5dda..0278b4b 100644
--- a/.github/workflows/docker-publish.yml
+++ b/.github/workflows/docker-publish.yml
@@ -6,28 +6,25 @@ on:
jobs:
lint:
- name: Pylint
runs-on: ubuntu-latest
permissions:
contents: read
+ strategy:
+ matrix:
+ python-version: ["3.12"]
steps:
- - name: Check out the repo
- uses: actions/checkout@v4
- with:
- persist-credentials: false
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: "3.12"
- cache: pip
-
- - name: Install dependencies
- run: pip install pylint
-
- - name: Run pylint
- run: pylint app/ main.py --output-format=colorized --exit-zero
- continue-on-error: true
+ - uses: actions/checkout@v4
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install pylint && pip install -r requirements.txt
+ - name: Analyse the code with pylint
+ run: |
+ pylint --rcfile=.pylintrc $(git ls-files '*.py' ':!migrations')
push_to_registry:
name: Push Docker image to Docker Hub
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
new file mode 100644
index 0000000..4d7382a
--- /dev/null
+++ b/.github/workflows/pylint.yml
@@ -0,0 +1,25 @@
+name: Pylint
+
+on: [push, pull_request]
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ strategy:
+ matrix:
+ python-version: ["3.12"]
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install pylint && pip install -r requirements.txt
+ - name: Analyse the code with pylint
+ run: |
+ pylint --rcfile=.pylintrc $(git ls-files '*.py' ':!migrations')
diff --git a/CLAUDE.md b/CLAUDE.md
index a629c53..31f8993 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -69,13 +69,13 @@ AppSetting (key/value table for all runtime config: scheduler, notifications, en
**Shop adapters** (`app/shop_adapters/`): Subclass `BaseAdapter`, set `domain` and optionally `fetch_engine = "cloudscraper"`, implement `extract(html, url) → AdapterResult`. Register in `registry.py` via `_reg(YourAdapter())`. Currently registered: `3djake.de`, `prusa3d.com`, `anycubic.com`, `eu.store.bambulab.com`, `esun3dstore.com`, `esun3dstoreeu.com`, `elegoo.com`.
-**Price parsing:** `parse_price()` in `routes/shop_rules.py` handles German (`1.299,00 €`) and English (`1,299.00`) formats, plus JSON-LD key fragments. Import from there when needed elsewhere.
+**Price parsing:** `parse_price()` in `routes/shop_rules.py` handles Currency formats, plus JSON-LD key fragments. Import from there when needed elsewhere.
**Notification settings:** Stored in `AppSetting` key/value rows. Access via `app/settings_service.py` (`get_all` / `set_many`). Send via `app/notification_service.py`. Keys: `discord.enabled`, `discord.webhook_url`, `smtp.*`.
## Key conventions
-**Language:** All code and user-facing text must be English — no Other Language words anywhere in this repo (open-source on GitHub). This includes flash messages, validation errors, log messages, notification/email/Discord copy, comments, and seed data. Conversation with the user may stay in German per their global preference, but nothing written to a file in this repo may contain German. Double-check every new or edited string before finishing a task — this has been missed before (e.g. `app/notification_service.py`, `app/alert_service.py` initially shipped with German copy and had to be fixed after the fact).
+**Language:** All code and user-facing text must be English — no Other Language words anywhere in this repo (open-source on GitHub). This includes flash messages, validation errors, log messages, notification/email/Discord copy, comments, and seed data.
**CSRF:** Every state-changing form needs ``. The `before_request` hook validates it on POST/PUT/PATCH/DELETE. JSON requests (content-type contains `json`) are exempt.
diff --git a/README.md b/README.md
index 4715731..53fd20d 100644
--- a/README.md
+++ b/README.md
@@ -219,7 +219,7 @@ To add a new adapter: subclass `BaseAdapter` in `app/shop_adapters/`, implement
### ShopRules (generic)
-For any other shop: create a rule at `/shop-rules` with domain, CSS price selector, and optional regex — or use the visual point-and-click picker to build one without touching CSS. German (`1.299,00 €`) and English (`1,299.00`) price formats are detected automatically.
+For any other shop: create a rule at `/shop-rules` with domain, CSS price selector, and optional regex — or use the visual point-and-click picker to build one without touching CSS.
---
diff --git a/app/price_check_service.py b/app/price_check_service.py
index 94d3951..eb6e138 100644
--- a/app/price_check_service.py
+++ b/app/price_check_service.py
@@ -28,12 +28,17 @@
_HTTPX_TIMEOUT = 12.0
-async def _fetch_httpx(url: str) -> tuple[str | None, str | None]:
+async def _fetch_httpx(
+ url: str, extra_headers: dict | None = None, warmup_url: str | None = None
+) -> tuple[str | None, str | None]:
import httpx
+ headers = {**_HTTPX_HEADERS, **extra_headers} if extra_headers else _HTTPX_HEADERS
try:
async with httpx.AsyncClient(
- follow_redirects=True, timeout=_HTTPX_TIMEOUT, headers=_HTTPX_HEADERS
+ follow_redirects=True, timeout=_HTTPX_TIMEOUT, headers=headers
) as client:
+ if warmup_url:
+ await client.get(warmup_url)
resp = await client.get(url)
resp.raise_for_status()
return resp.text, None
@@ -124,7 +129,11 @@ async def check_price(
elif effective_engine == "playwright":
html, fetch_error = await _fetch_playwright(link_url, settings)
else:
- html, fetch_error = await _fetch_httpx(link_url)
+ html, fetch_error = await _fetch_httpx(
+ link_url,
+ adapter.fetch_headers(link_url) if adapter else None,
+ adapter.warmup_url(link_url) if adapter else None,
+ )
if fetch_error:
await _save_error(link_id, link_currency, fetch_error)
diff --git a/app/routes/api_keys.py b/app/routes/api_keys.py
index f7c6a3a..e949b42 100644
--- a/app/routes/api_keys.py
+++ b/app/routes/api_keys.py
@@ -15,8 +15,8 @@
def admin_required(f):
@wraps(f)
async def wrapper(*args, **kwargs):
- async with get_db() as session:
- user = await session.get(User, int(current_user.auth_id))
+ async with get_db() as db:
+ user = await db.get(User, int(current_user.auth_id))
if not user or user.role != UserRole.admin:
abort(403)
return await f(*args, **kwargs)
@@ -67,11 +67,11 @@ async def create():
@login_required
@admin_required
async def delete(key_id: int):
- async with get_db() as session:
- key = await session.get(ApiKey, key_id)
+ async with get_db() as db:
+ key = await db.get(ApiKey, key_id)
if not key:
abort(404)
- await session.delete(key)
+ await db.delete(key)
await flash("API key revoked.", "success")
return redirect(url_for("api_keys.index"))
diff --git a/app/routes/shop_rules.py b/app/routes/shop_rules.py
index ac066e0..3e8aafe 100644
--- a/app/routes/shop_rules.py
+++ b/app/routes/shop_rules.py
@@ -41,7 +41,7 @@ async def wrapper(*args, **kwargs):
def parse_price(raw: str) -> float:
"""
- Parse price strings in German or English format to float.
+ Parse price strings format to float.
"17,99 €" → 17.99
"€17.99" → 17.99
"1.299,00 €" → 1299.00
@@ -61,10 +61,8 @@ def parse_price(raw: str) -> float:
comma_pos = s.rfind(",")
if comma_pos > dot_pos:
- # German: 1.299,00 — last separator is comma → decimal
s = s.replace(".", "").replace(",", ".")
elif dot_pos > comma_pos:
- # English or plain float: 1,299.00 or 32.990000 — last separator is dot
s = s.replace(",", "")
else:
s = s.replace(",", ".")
diff --git a/app/shop_adapters/_3djake.py b/app/shop_adapters/_3djake.py
index 37f32cb..26f4739 100644
--- a/app/shop_adapters/_3djake.py
+++ b/app/shop_adapters/_3djake.py
@@ -8,6 +8,7 @@
class ThreeDJakeAdapter(BaseAdapter):
domains = ("3djake.de",)
+ fetch_engine = "httpx"
_PRICE_SEL = ".price"
_PRICE_RE = r"\d+[,\.]\d{2}"
@@ -15,6 +16,13 @@ class ThreeDJakeAdapter(BaseAdapter):
_AVAIL_SEL = "[class*='availab']"
def extract(self, html: str, url: str) -> AdapterResult:
+ title_node = _extract(html, "title", None)
+ if title_node and "Request Problem" in title_node:
+ return AdapterResult(
+ status="blocked",
+ error_message="niceshops WAF blocked the request (Error 425 - Security Filtering).",
+ )
+
title = _extract(html, self._TITLE_SEL, None)
price_raw = _extract(html, self._PRICE_SEL, self._PRICE_RE)
availability = _extract(html, self._AVAIL_SEL, None)
diff --git a/app/shop_adapters/_amazon.py b/app/shop_adapters/_amazon.py
new file mode 100644
index 0000000..abd1713
--- /dev/null
+++ b/app/shop_adapters/_amazon.py
@@ -0,0 +1,89 @@
+"""
+Amazon adapter — all major Amazon marketplaces, confirmed working via plain httpx.
+"""
+from urllib.parse import urlparse
+
+from selectolax.parser import HTMLParser
+
+from app.routes.shop_rules import parse_price
+from .base import BaseAdapter, AdapterResult
+
+_ACCEPT_LANGUAGE = {
+ "amazon.com": "en-US,en;q=0.9",
+ "amazon.de": "de-DE,de;q=0.9,en;q=0.8",
+ "amazon.co.uk": "en-GB,en;q=0.9",
+ "amazon.fr": "fr-FR,fr;q=0.9,en;q=0.8",
+ "amazon.it": "it-IT,it;q=0.9,en;q=0.8",
+ "amazon.es": "es-ES,es;q=0.9,en;q=0.8",
+ "amazon.nl": "nl-NL,nl;q=0.9,en;q=0.8",
+ "amazon.se": "sv-SE,sv;q=0.9,en;q=0.8",
+ "amazon.pl": "pl-PL,pl;q=0.9,en;q=0.8",
+ "amazon.co.jp": "ja-JP,ja;q=0.9,en;q=0.8",
+ "amazon.ca": "en-CA,en;q=0.9",
+ "amazon.com.au": "en-AU,en;q=0.9",
+ "amazon.in": "en-IN,en;q=0.9",
+}
+
+
+class AmazonAdapter(BaseAdapter):
+ domains = tuple(_ACCEPT_LANGUAGE)
+ fetch_engine = "httpx"
+
+ def fetch_headers(self, url: str) -> dict | None:
+ domain = (urlparse(url).hostname or "").removeprefix("www.")
+ lang = _ACCEPT_LANGUAGE.get(domain)
+ return {"Accept-Language": lang} if lang else None
+
+ def extract(self, html: str, url: str) -> AdapterResult:
+ tree = HTMLParser(html)
+
+ is_captcha = (
+ tree.css_first("form[action*='validateCaptcha']") is not None
+ or "/errors/validateCaptcha" in url
+ or "Enter the characters you see below" in html
+ or "Type the characters you see in this image" in html
+ )
+ if is_captcha:
+ return AdapterResult(
+ status="blocked",
+ error_message="Amazon returned a CAPTCHA challenge page.",
+ )
+
+ title_node = tree.css_first("#productTitle")
+ title = title_node.text(strip=True) if title_node else None
+
+ price_node = (
+ tree.css_first(".a-price .a-offscreen")
+ or tree.css_first("#priceblock_ourprice")
+ or tree.css_first("#priceblock_dealprice")
+ )
+ price_raw = price_node.text(strip=True) if price_node else None
+
+ if not price_raw:
+ return AdapterResult(
+ status="error",
+ error_message="Price element not found — page structure may have changed or product is unavailable.",
+ title=title,
+ )
+
+ try:
+ price_parsed = parse_price(price_raw)
+ except ValueError as e:
+ return AdapterResult(
+ status="error",
+ price_raw=price_raw,
+ error_message=f"Price parse failed: {price_raw!r} → {e}",
+ title=title,
+ )
+
+ avail_node = tree.css_first("#availability .primary-availability-message") \
+ or tree.css_first("#availability span")
+ availability = avail_node.text(strip=True) if avail_node else None
+
+ return AdapterResult(
+ status="success",
+ price_raw=price_raw,
+ price_parsed=price_parsed,
+ availability=availability,
+ title=title,
+ )
diff --git a/app/shop_adapters/_anycubic.py b/app/shop_adapters/_anycubic.py
index 2367705..da003e6 100644
--- a/app/shop_adapters/_anycubic.py
+++ b/app/shop_adapters/_anycubic.py
@@ -18,6 +18,7 @@
class AnycubicAdapter(BaseAdapter):
domains = ("anycubic.com",)
+ fetch_engine = "httpx"
def extract(self, html: str, url: str) -> AdapterResult:
title = _extract(html, "h1", None)
diff --git a/app/shop_adapters/_ebay.py b/app/shop_adapters/_ebay.py
index 7b50a1f..1c6c1a8 100644
--- a/app/shop_adapters/_ebay.py
+++ b/app/shop_adapters/_ebay.py
@@ -1,19 +1,54 @@
"""
-eBay adapter — ebay.de item pages.
-Price from .x-price-primary (main listing price text).
-eBay may block httpx/cloudscraper — use Playwright engine.
-If eBay returns an error page, status is set to 'blocked'.
+eBay adapter — all major eBay marketplaces, plain httpx (mirrors the Amazon
+adapter approach: a normal browser User-Agent + Accept-Language sails through
+without triggering the interstitial/CAPTCHA that Playwright and cloudscraper's
+TLS fingerprint tend to trip on eBay).
"""
import re
+from urllib.parse import urlparse
+
from selectolax.parser import HTMLParser
+
from app.routes.shop_rules import parse_price
from .base import BaseAdapter, AdapterResult
-_PRICE_RE = re.compile(r'[\d.,]+')
+_ACCEPT_LANGUAGE = {
+ "ebay.com": "en-US,en;q=0.9",
+ "ebay.de": "de-DE,de;q=0.9,en;q=0.8",
+ "ebay.co.uk": "en-GB,en;q=0.9",
+ "ebay.fr": "fr-FR,fr;q=0.9,en;q=0.8",
+ "ebay.it": "it-IT,it;q=0.9,en;q=0.8",
+ "ebay.es": "es-ES,es;q=0.9,en;q=0.8",
+ "ebay.at": "de-AT,de;q=0.9,en;q=0.8",
+ "ebay.nl": "nl-NL,nl;q=0.9,en;q=0.8",
+ "ebay.ie": "en-IE,en;q=0.9",
+ "ebay.pl": "pl-PL,pl;q=0.9,en;q=0.8",
+ "ebay.ch": "de-CH,de;q=0.9,fr;q=0.8,en;q=0.7",
+ "ebay.ca": "en-CA,en;q=0.9",
+ "ebay.com.au": "en-AU,en;q=0.9",
+ "ebay.com.hk": "en-HK,en;q=0.9",
+ "ebay.com.sg": "en-SG,en;q=0.9",
+ "ebay.com.my": "en-MY,en;q=0.9",
+ "ebay.ph": "en-PH,en;q=0.9",
+}
+
+_PRICE_RE = re.compile(r"[\d.,]+")
class EbayAdapter(BaseAdapter):
- domains = ("ebay.de",)
+ domains = tuple(_ACCEPT_LANGUAGE)
+ fetch_engine = "httpx"
+
+ def fetch_headers(self, url: str) -> dict | None:
+ domain = (urlparse(url).hostname or "").removeprefix("www.")
+ lang = _ACCEPT_LANGUAGE.get(domain)
+ return {"Accept-Language": lang} if lang else None
+
+ def warmup_url(self, url: str) -> str | None:
+ # A cold request straight to an item page gets eBay's generic error
+ # page — GET the homepage first (same client) to pick up session cookies.
+ domain = (urlparse(url).hostname or "").removeprefix("www.")
+ return f"https://www.{domain}/" if domain in _ACCEPT_LANGUAGE else None
def extract(self, html: str, url: str) -> AdapterResult:
tree = HTMLParser(html)
@@ -21,23 +56,19 @@ def extract(self, html: str, url: str) -> AdapterResult:
title_node = tree.css_first("h1")
title = title_node.text(strip=True) if title_node else None
- # Detect eBay error/blocked page
page_title_node = tree.css_first("title")
page_title = page_title_node.text(strip=True) if page_title_node else ""
- if "error page" in page_title.lower() or "sorry" in page_title.lower():
+ if tree.css_first("#captcha") or "error page" in page_title.lower() or "sorry" in page_title.lower():
return AdapterResult(
status="blocked",
- error_message="eBay returned an error page — request was blocked or item unavailable",
+ error_message="eBay returned an error/CAPTCHA page.",
title=title,
)
- # Primary price: .x-price-primary contains the BIN / current bid price
price_node = tree.css_first(".x-price-primary")
if price_node:
- # Strip currency symbol and whitespace, keep only digits/separators
price_text = price_node.text(strip=True)
else:
- # Fallback: [itemprop="price"] or og:price
itemprop = tree.css_first('[itemprop="price"]')
if itemprop:
price_text = itemprop.attributes.get("content") or itemprop.text(strip=True)
@@ -48,7 +79,7 @@ def extract(self, html: str, url: str) -> AdapterResult:
if not price_text:
return AdapterResult(
status="error",
- error_message=".x-price-primary not found — eBay DOM may have changed or item is unavailable",
+ error_message="Price element not found — page structure may have changed or item is unavailable.",
title=title,
)
@@ -63,18 +94,17 @@ def extract(self, html: str, url: str) -> AdapterResult:
try:
price_parsed = parse_price(m.group(0))
- except (ValueError, AttributeError) as e:
+ except ValueError as e:
return AdapterResult(
status="error",
price_raw=price_text,
- error_message=f"parse_price failed: {price_text!r} → {e}",
+ error_message=f"Price parse failed: {price_text!r} → {e}",
title=title,
)
- # Availability
avail_node = (
- tree.css_first(".d-quantity__availability")
- or tree.css_first("[data-testid='ux-seller-section__item--seller-info']")
+ tree.css_first(".d-quantity__availability")
+ or tree.css_first("[data-testid='ux-seller-section__item--seller-info']")
)
availability = avail_node.text(strip=True) if avail_node else None
diff --git a/app/shop_adapters/_elegoo.py b/app/shop_adapters/_elegoo.py
index 32db0d4..98df523 100644
--- a/app/shop_adapters/_elegoo.py
+++ b/app/shop_adapters/_elegoo.py
@@ -2,6 +2,8 @@
Elegoo Shop adapter — Shopify store, confirmed working 2026-06-30.
Price from (already decimal, USD).
Availability from JSON-LD schema.org/InStock|OutOfStock.
+Plain httpx works fine (~2MB SSR page, including regional subdomains like
+de.elegoo.com), no need for Playwright.
"""
import re
from selectolax.parser import HTMLParser
@@ -17,6 +19,7 @@
class ElegooAdapter(BaseAdapter):
domains = ("elegoo.com",)
+ fetch_engine = "httpx"
def extract(self, html: str, url: str) -> AdapterResult:
tree = HTMLParser(html)
diff --git a/app/shop_adapters/_esun.py b/app/shop_adapters/_esun.py
index 2878823..23e3a56 100644
--- a/app/shop_adapters/_esun.py
+++ b/app/shop_adapters/_esun.py
@@ -3,7 +3,6 @@
esun3dstore.com — USD store, confirmed 2026-06-30
esun3dstoreeu.com — EUR store, confirmed 2026-06-30
-Note: esun3d.com is the brand/marketing site — prices are JS-rendered there.
"""
import re
diff --git a/app/shop_adapters/_prusa.py b/app/shop_adapters/_prusa.py
index ff50838..1090735 100644
--- a/app/shop_adapters/_prusa.py
+++ b/app/shop_adapters/_prusa.py
@@ -1,6 +1,7 @@
"""
Prusa Shop adapter — WooCommerce + JSON-LD, confirmed working 2026-06-29.
Price extracted from