diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index dc149bd..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: CI - -on: - push: - branches: ["**"] - pull_request: - branches: ["**"] - -jobs: - lint: - name: Pylint - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - name: Check out the repo - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: pip - - - name: Install dependencies - run: pip install pylint - - - name: Run pylint - run: pylint app/ main.py --output-format=colorized --exit-zero - continue-on-error: true diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9eb5dda..0278b4b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -6,28 +6,25 @@ on: jobs: lint: - name: Pylint runs-on: ubuntu-latest permissions: contents: read + strategy: + matrix: + python-version: ["3.12"] steps: - - name: Check out the repo - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: pip - - - name: Install dependencies - run: pip install pylint - - - name: Run pylint - run: pylint app/ main.py --output-format=colorized --exit-zero - continue-on-error: true + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint && pip install -r requirements.txt + - name: Analyse the code with pylint + run: | + pylint --rcfile=.pylintrc $(git ls-files '*.py' ':!migrations') push_to_registry: name: Push Docker image to Docker Hub diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..4d7382a --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,25 @@ +name: Pylint + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + matrix: + python-version: ["3.12"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint && pip install -r requirements.txt + - name: Analyse the code with pylint + run: | + pylint --rcfile=.pylintrc $(git ls-files '*.py' ':!migrations') diff --git a/CLAUDE.md b/CLAUDE.md index a629c53..31f8993 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,13 +69,13 @@ AppSetting (key/value table for all runtime config: scheduler, notifications, en **Shop adapters** (`app/shop_adapters/`): Subclass `BaseAdapter`, set `domain` and optionally `fetch_engine = "cloudscraper"`, implement `extract(html, url) → AdapterResult`. Register in `registry.py` via `_reg(YourAdapter())`. Currently registered: `3djake.de`, `prusa3d.com`, `anycubic.com`, `eu.store.bambulab.com`, `esun3dstore.com`, `esun3dstoreeu.com`, `elegoo.com`. -**Price parsing:** `parse_price()` in `routes/shop_rules.py` handles German (`1.299,00 €`) and English (`1,299.00`) formats, plus JSON-LD key fragments. Import from there when needed elsewhere. +**Price parsing:** `parse_price()` in `routes/shop_rules.py` handles Currency formats, plus JSON-LD key fragments. Import from there when needed elsewhere. **Notification settings:** Stored in `AppSetting` key/value rows. Access via `app/settings_service.py` (`get_all` / `set_many`). Send via `app/notification_service.py`. Keys: `discord.enabled`, `discord.webhook_url`, `smtp.*`. ## Key conventions -**Language:** All code and user-facing text must be English — no Other Language words anywhere in this repo (open-source on GitHub). This includes flash messages, validation errors, log messages, notification/email/Discord copy, comments, and seed data. Conversation with the user may stay in German per their global preference, but nothing written to a file in this repo may contain German. Double-check every new or edited string before finishing a task — this has been missed before (e.g. `app/notification_service.py`, `app/alert_service.py` initially shipped with German copy and had to be fixed after the fact). +**Language:** All code and user-facing text must be English — no Other Language words anywhere in this repo (open-source on GitHub). This includes flash messages, validation errors, log messages, notification/email/Discord copy, comments, and seed data. **CSRF:** Every state-changing form needs ``. The `before_request` hook validates it on POST/PUT/PATCH/DELETE. JSON requests (content-type contains `json`) are exempt. diff --git a/README.md b/README.md index 4715731..53fd20d 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ To add a new adapter: subclass `BaseAdapter` in `app/shop_adapters/`, implement ### ShopRules (generic) -For any other shop: create a rule at `/shop-rules` with domain, CSS price selector, and optional regex — or use the visual point-and-click picker to build one without touching CSS. German (`1.299,00 €`) and English (`1,299.00`) price formats are detected automatically. +For any other shop: create a rule at `/shop-rules` with domain, CSS price selector, and optional regex — or use the visual point-and-click picker to build one without touching CSS. --- diff --git a/app/price_check_service.py b/app/price_check_service.py index 94d3951..eb6e138 100644 --- a/app/price_check_service.py +++ b/app/price_check_service.py @@ -28,12 +28,17 @@ _HTTPX_TIMEOUT = 12.0 -async def _fetch_httpx(url: str) -> tuple[str | None, str | None]: +async def _fetch_httpx( + url: str, extra_headers: dict | None = None, warmup_url: str | None = None +) -> tuple[str | None, str | None]: import httpx + headers = {**_HTTPX_HEADERS, **extra_headers} if extra_headers else _HTTPX_HEADERS try: async with httpx.AsyncClient( - follow_redirects=True, timeout=_HTTPX_TIMEOUT, headers=_HTTPX_HEADERS + follow_redirects=True, timeout=_HTTPX_TIMEOUT, headers=headers ) as client: + if warmup_url: + await client.get(warmup_url) resp = await client.get(url) resp.raise_for_status() return resp.text, None @@ -124,7 +129,11 @@ async def check_price( elif effective_engine == "playwright": html, fetch_error = await _fetch_playwright(link_url, settings) else: - html, fetch_error = await _fetch_httpx(link_url) + html, fetch_error = await _fetch_httpx( + link_url, + adapter.fetch_headers(link_url) if adapter else None, + adapter.warmup_url(link_url) if adapter else None, + ) if fetch_error: await _save_error(link_id, link_currency, fetch_error) diff --git a/app/routes/api_keys.py b/app/routes/api_keys.py index f7c6a3a..e949b42 100644 --- a/app/routes/api_keys.py +++ b/app/routes/api_keys.py @@ -15,8 +15,8 @@ def admin_required(f): @wraps(f) async def wrapper(*args, **kwargs): - async with get_db() as session: - user = await session.get(User, int(current_user.auth_id)) + async with get_db() as db: + user = await db.get(User, int(current_user.auth_id)) if not user or user.role != UserRole.admin: abort(403) return await f(*args, **kwargs) @@ -67,11 +67,11 @@ async def create(): @login_required @admin_required async def delete(key_id: int): - async with get_db() as session: - key = await session.get(ApiKey, key_id) + async with get_db() as db: + key = await db.get(ApiKey, key_id) if not key: abort(404) - await session.delete(key) + await db.delete(key) await flash("API key revoked.", "success") return redirect(url_for("api_keys.index")) diff --git a/app/routes/shop_rules.py b/app/routes/shop_rules.py index ac066e0..3e8aafe 100644 --- a/app/routes/shop_rules.py +++ b/app/routes/shop_rules.py @@ -41,7 +41,7 @@ async def wrapper(*args, **kwargs): def parse_price(raw: str) -> float: """ - Parse price strings in German or English format to float. + Parse price strings format to float. "17,99 €" → 17.99 "€17.99" → 17.99 "1.299,00 €" → 1299.00 @@ -61,10 +61,8 @@ def parse_price(raw: str) -> float: comma_pos = s.rfind(",") if comma_pos > dot_pos: - # German: 1.299,00 — last separator is comma → decimal s = s.replace(".", "").replace(",", ".") elif dot_pos > comma_pos: - # English or plain float: 1,299.00 or 32.990000 — last separator is dot s = s.replace(",", "") else: s = s.replace(",", ".") diff --git a/app/shop_adapters/_3djake.py b/app/shop_adapters/_3djake.py index 37f32cb..26f4739 100644 --- a/app/shop_adapters/_3djake.py +++ b/app/shop_adapters/_3djake.py @@ -8,6 +8,7 @@ class ThreeDJakeAdapter(BaseAdapter): domains = ("3djake.de",) + fetch_engine = "httpx" _PRICE_SEL = ".price" _PRICE_RE = r"\d+[,\.]\d{2}" @@ -15,6 +16,13 @@ class ThreeDJakeAdapter(BaseAdapter): _AVAIL_SEL = "[class*='availab']" def extract(self, html: str, url: str) -> AdapterResult: + title_node = _extract(html, "title", None) + if title_node and "Request Problem" in title_node: + return AdapterResult( + status="blocked", + error_message="niceshops WAF blocked the request (Error 425 - Security Filtering).", + ) + title = _extract(html, self._TITLE_SEL, None) price_raw = _extract(html, self._PRICE_SEL, self._PRICE_RE) availability = _extract(html, self._AVAIL_SEL, None) diff --git a/app/shop_adapters/_amazon.py b/app/shop_adapters/_amazon.py new file mode 100644 index 0000000..abd1713 --- /dev/null +++ b/app/shop_adapters/_amazon.py @@ -0,0 +1,89 @@ +""" +Amazon adapter — all major Amazon marketplaces, confirmed working via plain httpx. +""" +from urllib.parse import urlparse + +from selectolax.parser import HTMLParser + +from app.routes.shop_rules import parse_price +from .base import BaseAdapter, AdapterResult + +_ACCEPT_LANGUAGE = { + "amazon.com": "en-US,en;q=0.9", + "amazon.de": "de-DE,de;q=0.9,en;q=0.8", + "amazon.co.uk": "en-GB,en;q=0.9", + "amazon.fr": "fr-FR,fr;q=0.9,en;q=0.8", + "amazon.it": "it-IT,it;q=0.9,en;q=0.8", + "amazon.es": "es-ES,es;q=0.9,en;q=0.8", + "amazon.nl": "nl-NL,nl;q=0.9,en;q=0.8", + "amazon.se": "sv-SE,sv;q=0.9,en;q=0.8", + "amazon.pl": "pl-PL,pl;q=0.9,en;q=0.8", + "amazon.co.jp": "ja-JP,ja;q=0.9,en;q=0.8", + "amazon.ca": "en-CA,en;q=0.9", + "amazon.com.au": "en-AU,en;q=0.9", + "amazon.in": "en-IN,en;q=0.9", +} + + +class AmazonAdapter(BaseAdapter): + domains = tuple(_ACCEPT_LANGUAGE) + fetch_engine = "httpx" + + def fetch_headers(self, url: str) -> dict | None: + domain = (urlparse(url).hostname or "").removeprefix("www.") + lang = _ACCEPT_LANGUAGE.get(domain) + return {"Accept-Language": lang} if lang else None + + def extract(self, html: str, url: str) -> AdapterResult: + tree = HTMLParser(html) + + is_captcha = ( + tree.css_first("form[action*='validateCaptcha']") is not None + or "/errors/validateCaptcha" in url + or "Enter the characters you see below" in html + or "Type the characters you see in this image" in html + ) + if is_captcha: + return AdapterResult( + status="blocked", + error_message="Amazon returned a CAPTCHA challenge page.", + ) + + title_node = tree.css_first("#productTitle") + title = title_node.text(strip=True) if title_node else None + + price_node = ( + tree.css_first(".a-price .a-offscreen") + or tree.css_first("#priceblock_ourprice") + or tree.css_first("#priceblock_dealprice") + ) + price_raw = price_node.text(strip=True) if price_node else None + + if not price_raw: + return AdapterResult( + status="error", + error_message="Price element not found — page structure may have changed or product is unavailable.", + title=title, + ) + + try: + price_parsed = parse_price(price_raw) + except ValueError as e: + return AdapterResult( + status="error", + price_raw=price_raw, + error_message=f"Price parse failed: {price_raw!r} → {e}", + title=title, + ) + + avail_node = tree.css_first("#availability .primary-availability-message") \ + or tree.css_first("#availability span") + availability = avail_node.text(strip=True) if avail_node else None + + return AdapterResult( + status="success", + price_raw=price_raw, + price_parsed=price_parsed, + availability=availability, + title=title, + ) diff --git a/app/shop_adapters/_anycubic.py b/app/shop_adapters/_anycubic.py index 2367705..da003e6 100644 --- a/app/shop_adapters/_anycubic.py +++ b/app/shop_adapters/_anycubic.py @@ -18,6 +18,7 @@ class AnycubicAdapter(BaseAdapter): domains = ("anycubic.com",) + fetch_engine = "httpx" def extract(self, html: str, url: str) -> AdapterResult: title = _extract(html, "h1", None) diff --git a/app/shop_adapters/_ebay.py b/app/shop_adapters/_ebay.py index 7b50a1f..1c6c1a8 100644 --- a/app/shop_adapters/_ebay.py +++ b/app/shop_adapters/_ebay.py @@ -1,19 +1,54 @@ """ -eBay adapter — ebay.de item pages. -Price from .x-price-primary (main listing price text). -eBay may block httpx/cloudscraper — use Playwright engine. -If eBay returns an error page, status is set to 'blocked'. +eBay adapter — all major eBay marketplaces, plain httpx (mirrors the Amazon +adapter approach: a normal browser User-Agent + Accept-Language sails through +without triggering the interstitial/CAPTCHA that Playwright and cloudscraper's +TLS fingerprint tend to trip on eBay). """ import re +from urllib.parse import urlparse + from selectolax.parser import HTMLParser + from app.routes.shop_rules import parse_price from .base import BaseAdapter, AdapterResult -_PRICE_RE = re.compile(r'[\d.,]+') +_ACCEPT_LANGUAGE = { + "ebay.com": "en-US,en;q=0.9", + "ebay.de": "de-DE,de;q=0.9,en;q=0.8", + "ebay.co.uk": "en-GB,en;q=0.9", + "ebay.fr": "fr-FR,fr;q=0.9,en;q=0.8", + "ebay.it": "it-IT,it;q=0.9,en;q=0.8", + "ebay.es": "es-ES,es;q=0.9,en;q=0.8", + "ebay.at": "de-AT,de;q=0.9,en;q=0.8", + "ebay.nl": "nl-NL,nl;q=0.9,en;q=0.8", + "ebay.ie": "en-IE,en;q=0.9", + "ebay.pl": "pl-PL,pl;q=0.9,en;q=0.8", + "ebay.ch": "de-CH,de;q=0.9,fr;q=0.8,en;q=0.7", + "ebay.ca": "en-CA,en;q=0.9", + "ebay.com.au": "en-AU,en;q=0.9", + "ebay.com.hk": "en-HK,en;q=0.9", + "ebay.com.sg": "en-SG,en;q=0.9", + "ebay.com.my": "en-MY,en;q=0.9", + "ebay.ph": "en-PH,en;q=0.9", +} + +_PRICE_RE = re.compile(r"[\d.,]+") class EbayAdapter(BaseAdapter): - domains = ("ebay.de",) + domains = tuple(_ACCEPT_LANGUAGE) + fetch_engine = "httpx" + + def fetch_headers(self, url: str) -> dict | None: + domain = (urlparse(url).hostname or "").removeprefix("www.") + lang = _ACCEPT_LANGUAGE.get(domain) + return {"Accept-Language": lang} if lang else None + + def warmup_url(self, url: str) -> str | None: + # A cold request straight to an item page gets eBay's generic error + # page — GET the homepage first (same client) to pick up session cookies. + domain = (urlparse(url).hostname or "").removeprefix("www.") + return f"https://www.{domain}/" if domain in _ACCEPT_LANGUAGE else None def extract(self, html: str, url: str) -> AdapterResult: tree = HTMLParser(html) @@ -21,23 +56,19 @@ def extract(self, html: str, url: str) -> AdapterResult: title_node = tree.css_first("h1") title = title_node.text(strip=True) if title_node else None - # Detect eBay error/blocked page page_title_node = tree.css_first("title") page_title = page_title_node.text(strip=True) if page_title_node else "" - if "error page" in page_title.lower() or "sorry" in page_title.lower(): + if tree.css_first("#captcha") or "error page" in page_title.lower() or "sorry" in page_title.lower(): return AdapterResult( status="blocked", - error_message="eBay returned an error page — request was blocked or item unavailable", + error_message="eBay returned an error/CAPTCHA page.", title=title, ) - # Primary price: .x-price-primary contains the BIN / current bid price price_node = tree.css_first(".x-price-primary") if price_node: - # Strip currency symbol and whitespace, keep only digits/separators price_text = price_node.text(strip=True) else: - # Fallback: [itemprop="price"] or og:price itemprop = tree.css_first('[itemprop="price"]') if itemprop: price_text = itemprop.attributes.get("content") or itemprop.text(strip=True) @@ -48,7 +79,7 @@ def extract(self, html: str, url: str) -> AdapterResult: if not price_text: return AdapterResult( status="error", - error_message=".x-price-primary not found — eBay DOM may have changed or item is unavailable", + error_message="Price element not found — page structure may have changed or item is unavailable.", title=title, ) @@ -63,18 +94,17 @@ def extract(self, html: str, url: str) -> AdapterResult: try: price_parsed = parse_price(m.group(0)) - except (ValueError, AttributeError) as e: + except ValueError as e: return AdapterResult( status="error", price_raw=price_text, - error_message=f"parse_price failed: {price_text!r} → {e}", + error_message=f"Price parse failed: {price_text!r} → {e}", title=title, ) - # Availability avail_node = ( - tree.css_first(".d-quantity__availability") - or tree.css_first("[data-testid='ux-seller-section__item--seller-info']") + tree.css_first(".d-quantity__availability") + or tree.css_first("[data-testid='ux-seller-section__item--seller-info']") ) availability = avail_node.text(strip=True) if avail_node else None diff --git a/app/shop_adapters/_elegoo.py b/app/shop_adapters/_elegoo.py index 32db0d4..98df523 100644 --- a/app/shop_adapters/_elegoo.py +++ b/app/shop_adapters/_elegoo.py @@ -2,6 +2,8 @@ Elegoo Shop adapter — Shopify store, confirmed working 2026-06-30. Price from (already decimal, USD). Availability from JSON-LD schema.org/InStock|OutOfStock. +Plain httpx works fine (~2MB SSR page, including regional subdomains like +de.elegoo.com), no need for Playwright. """ import re from selectolax.parser import HTMLParser @@ -17,6 +19,7 @@ class ElegooAdapter(BaseAdapter): domains = ("elegoo.com",) + fetch_engine = "httpx" def extract(self, html: str, url: str) -> AdapterResult: tree = HTMLParser(html) diff --git a/app/shop_adapters/_esun.py b/app/shop_adapters/_esun.py index 2878823..23e3a56 100644 --- a/app/shop_adapters/_esun.py +++ b/app/shop_adapters/_esun.py @@ -3,7 +3,6 @@ esun3dstore.com — USD store, confirmed 2026-06-30 esun3dstoreeu.com — EUR store, confirmed 2026-06-30 -Note: esun3d.com is the brand/marketing site — prices are JS-rendered there. """ import re diff --git a/app/shop_adapters/_prusa.py b/app/shop_adapters/_prusa.py index ff50838..1090735 100644 --- a/app/shop_adapters/_prusa.py +++ b/app/shop_adapters/_prusa.py @@ -1,6 +1,7 @@ """ Prusa Shop adapter — WooCommerce + JSON-LD, confirmed working 2026-06-29. Price extracted from