diff --git a/backend/foresight/repository/job.py b/backend/foresight/repository/job.py index b06ecfc..6e5a35c 100644 --- a/backend/foresight/repository/job.py +++ b/backend/foresight/repository/job.py @@ -83,3 +83,23 @@ async def last_succeeded_start( Job.params.contains(params), ) ) + + +async def latest_finished_job( + session: AsyncSession, + *, + hotel_id: int, + kind: JobKind, + params: dict[str, Any], + status: JobStatus | None = None, +) -> Job | None: + """The newest finished job of this kind whose params contain `params`.""" + stmt = select(Job).where( + Job.hotel_id == hotel_id, + Job.kind == kind, + Job.finished_at.is_not(None), + Job.params.contains(params), + ) + if status is not None: + stmt = stmt.where(Job.status == status) + return await session.scalar(stmt.order_by(Job.finished_at.desc()).limit(1)) diff --git a/backend/foresight/routes/job.py b/backend/foresight/routes/job.py index 602d3e5..b8536eb 100644 --- a/backend/foresight/routes/job.py +++ b/backend/foresight/routes/job.py @@ -3,6 +3,9 @@ Poll `GET /jobs/{id}` for status and progress. If a job stops or fails part-way, `resume_from` says where to pick up: send it to the same ingest endpoint as `start` again (or as `end` for a run walking backwards). + +`GET /hotels/{id}/freshness` reads the same jobs to say when each kind of data +last refreshed and whether it's stale. """ from datetime import date, datetime @@ -20,6 +23,7 @@ paginate, ) from foresight.services import job as job_service +from foresight.services.domain import freshness as freshness_service router = APIRouter(tags=["Jobs"]) @@ -52,6 +56,31 @@ class JobResponse(ResponseSchema): ) +class KindFreshnessResponse(ResponseSchema): + last_success: datetime | None = Field( + description="When it last succeeded, UTC; null if it never has" + ) + last_failure: datetime | None = Field( + description="When the latest attempt failed or stopped, UTC; null if " + "the latest attempt succeeded or it never ran" + ) + error: str | None = Field(description="Why the latest attempt failed; null if not") + stale: bool = Field( + description="Never succeeded, or the last success is too old: 1 day for " + "events and weather, 7 for hotel data, 30 for training" + ) + + +class FreshnessResponse(ResponseSchema): + events: KindFreshnessResponse = Field(description="Routine events refresh") + weather: KindFreshnessResponse = Field(description="Routine forecast refresh") + hotel_data: KindFreshnessResponse = Field(description="Hotel data upload (CSV)") + training: KindFreshnessResponse = Field(description="Pricing model training") + recommendations: KindFreshnessResponse | None = Field( + description="Recommendation snapshots; always null until they're stored" + ) + + JobId = Annotated[int, Path(description="The job's id", examples=[1])] HotelId = Annotated[int, Path(description="The hotel's id", examples=[1])] @@ -76,3 +105,10 @@ async def list_jobs( session, hotel_id, limit=page.limit, offset=page.offset ) return paginate(JobResponse, jobs, total, page) + + +@router.get("/hotels/{hotel_id}/freshness", response_model=FreshnessResponse) +async def get_freshness(hotel_id: HotelId, session: SessionDep) -> FreshnessResponse: + """When each kind of data last refreshed, and whether it's stale.""" + result = await freshness_service.get_freshness(session, hotel_id) + return FreshnessResponse.model_validate(result) diff --git a/backend/foresight/services/domain/freshness.py b/backend/foresight/services/domain/freshness.py new file mode 100644 index 0000000..0dfbfec --- /dev/null +++ b/backend/foresight/services/domain/freshness.py @@ -0,0 +1,73 @@ +"""When each kind of data last refreshed for a hotel, and whether it's stale. + +Read from `jobs`; nothing is stored. Only routine refreshes count for events +and weather: a backfill of last year doesn't make upcoming rates fresh. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING + +from foresight.database.enums import JobKind, JobStatus +from foresight.repository import job as job_repo +from foresight.services.domain.hotel import get_hotel + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +# name: (job kind, params a run must have, how old it may get) +KINDS = { + "events": (JobKind.EVENTS, {"mode": "recent"}, timedelta(days=1)), + "weather": (JobKind.WEATHER, {"mode": "forecast"}, timedelta(days=1)), + "hotel_data": (JobKind.PERFORMANCE, {}, timedelta(days=7)), + "training": (JobKind.TRAIN, {}, timedelta(days=30)), +} + + +@dataclass(frozen=True, slots=True) +class Freshness: + last_success: datetime | None # None if it never succeeded + last_failure: datetime | None # set only if the latest attempt failed + error: str | None # why the latest attempt failed + stale: bool + + +@dataclass(frozen=True, slots=True) +class HotelFreshness: + """One hotel: every kind of data, keyed as in `KINDS`.""" + + events: Freshness + weather: Freshness + hotel_data: Freshness + training: Freshness + recommendations: Freshness | None = None # null until snapshots are stored + + +async def get_freshness(session: AsyncSession, hotel_id: int) -> HotelFreshness: + """Raises NotFoundError for an unknown hotel.""" + await get_hotel(session, hotel_id) + now = datetime.now(UTC) + result: dict[str, Freshness] = {} + for name, (kind, params, max_age) in KINDS.items(): + latest = await job_repo.latest_finished_job( + session, hotel_id=hotel_id, kind=kind, params=params + ) + failed = latest is not None and latest.status != JobStatus.SUCCEEDED + success = latest + if failed: + success = await job_repo.latest_finished_job( + session, + hotel_id=hotel_id, + kind=kind, + params=params, + status=JobStatus.SUCCEEDED, + ) + result[name] = Freshness( + last_success=success.finished_at if success else None, + last_failure=latest.finished_at if failed else None, + error=latest.error if failed else None, + stale=success is None or now - success.finished_at > max_age, + ) + return HotelFreshness(**result) diff --git a/backend/tests/test_freshness.py b/backend/tests/test_freshness.py new file mode 100644 index 0000000..91f6856 --- /dev/null +++ b/backend/tests/test_freshness.py @@ -0,0 +1,108 @@ +"""Freshness for a hotel whose jobs are a list in memory (no database).""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock + +from foresight.database.enums import JobKind, JobStatus +from foresight.database.models.job import Job +from foresight.services.domain import freshness + +NOW = datetime.now(UTC) + + +def _job( + status: JobStatus, + hours_ago: float, + error: str | None = None, + params: dict | None = None, +) -> Job: + return Job( + kind=JobKind.EVENTS, + status=status, + params={"mode": "recent"} if params is None else params, + finished_at=NOW - timedelta(hours=hours_ago), + error=error, + ) + + +async def _events(monkeypatch, jobs: list[Job]) -> freshness.Freshness: + async def latest(_session, *, kind, params, status=None, **_): + found = [ + j + for j in jobs + if j.kind == kind + and status in (None, j.status) + and params.items() <= j.params.items() + ] + return max(found, key=lambda j: j.finished_at, default=None) + + monkeypatch.setattr(freshness, "get_hotel", AsyncMock()) + monkeypatch.setattr(freshness.job_repo, "latest_finished_job", latest) + return (await freshness.get_freshness(None, 1)).events + + +async def test_never_run(monkeypatch) -> None: + events = await _events(monkeypatch, []) + assert events.last_success is None + assert events.last_failure is None + assert events.stale + + +async def test_succeeded(monkeypatch) -> None: + events = await _events(monkeypatch, [_job(JobStatus.SUCCEEDED, 2)]) + assert events.last_success is not None + assert events.last_failure is None + assert not events.stale + + +async def test_failed_after_success_shows_both(monkeypatch) -> None: + jobs = [_job(JobStatus.SUCCEEDED, 30), _job(JobStatus.FAILED, 1, "HTTP 500")] + events = await _events(monkeypatch, jobs) + assert events.last_success == jobs[0].finished_at + assert events.last_failure == jobs[1].finished_at + assert events.error == "HTTP 500" + assert events.stale # last success was 30 hours ago + + +async def test_failed_and_never_succeeded(monkeypatch) -> None: + failed = _job(JobStatus.FAILED, 1, "HTTP 500") + events = await _events(monkeypatch, [failed]) + assert events.last_success is None + assert events.last_failure == failed.finished_at + assert events.error == "HTTP 500" + assert events.stale # never succeeded + + +async def test_success_after_failure_clears_the_failure(monkeypatch) -> None: + jobs = [_job(JobStatus.FAILED, 5, "HTTP 500"), _job(JobStatus.SUCCEEDED, 1)] + events = await _events(monkeypatch, jobs) + assert events.last_success == jobs[1].finished_at + assert events.last_failure is None # the latest attempt didn't fail + assert events.error is None + assert not events.stale + + +async def test_succeeded_but_stale(monkeypatch) -> None: + old = _job(JobStatus.SUCCEEDED, 25) # events go stale after 24 hours + events = await _events(monkeypatch, [old]) + assert events.last_success == old.finished_at + assert events.last_failure is None + assert events.stale + + +async def test_backfill_does_not_count_as_a_refresh(monkeypatch) -> None: + backfill = _job(JobStatus.SUCCEEDED, 1, params={"mode": "window"}) + events = await _events(monkeypatch, [backfill]) + assert events.last_success is None # only routine refreshes count + assert events.stale + + +async def test_recommendations_are_not_tracked_yet(monkeypatch) -> None: + monkeypatch.setattr(freshness, "get_hotel", AsyncMock()) + monkeypatch.setattr( + freshness.job_repo, "latest_finished_job", AsyncMock(return_value=None) + ) + hotel = await freshness.get_freshness(None, 1) + assert hotel.recommendations is None diff --git a/docs/decisions/0033-data-freshness-rules.md b/docs/decisions/0033-data-freshness-rules.md new file mode 100644 index 0000000..3fd8673 --- /dev/null +++ b/docs/decisions/0033-data-freshness-rules.md @@ -0,0 +1,28 @@ +# 0033. Freshness comes from jobs; the service decides what's stale + +**Status:** Accepted ยท **Date:** 2026-10-26 + +## Context +A manager can't tell whether rates are fresh. If an events refresh or an +upload failed, nothing said so, and the frontend shouldn't encode the rules +for how old each kind of data may get. + +## Decision +- `GET /hotels/{id}/freshness` reads the newest finished jobs of each kind; + there is no new table. +- Only routine refreshes count: events `mode: "recent"`, weather + `mode: "forecast"`. Backfills don't make upcoming rates fresh. Any + successful hotel data upload or training run counts. +- Stopped and failed both count as a failed attempt. A failure is shown only + when it is the latest attempt; a later success clears it. +- Stale: events and weather after 1 day, hotel data after 7, training after + 30. Never succeeding is stale. The rules are constants (`KINDS`) in + `services/domain/freshness.py`. +- `recommendations` is always null until recommendation snapshots are stored; then + it gets a row in `KINDS`. + + +## Consequences +- Changing a threshold is a one-line change, with no frontend release. +- A weather backfill that also fetched the forecast doesn't count as a refresh. +- A failure followed by a success no longer shows; the job list still has it. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 00e33f1..cf2e62f 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -53,3 +53,4 @@ Number files sequentially (`0027-short-title.md`) and add them to the index. | [0030](0030-schema-descriptions-and-examples.md) | Every field is described; examples are built from the fields | API | | [0031](0031-pms-reports-for-training.md) | Guestline reports used for training | data | | [0032](0032-events-on-a-night-follow-the-model.md) | The events shown for a night follow the model's rule | API | +| [0033](0033-data-freshness-rules.md) | Freshness comes from jobs; the service decides what's stale | API | diff --git a/docs/system-design.md b/docs/system-design.md index 6ca585c..294498b 100644 --- a/docs/system-design.md +++ b/docs/system-design.md @@ -92,7 +92,7 @@ backend/foresight/ errors.py NotFound / Conflict / InvalidRequest / NotConfigured job.py job kinds, validation, handlers, lifecycle domain/ city, hotel, weather, weather_coverage, event, flight, - model_inputs, night, pricing_model, recommendation + model_inputs, night, pricing_model, recommendation, freshness ingest/ weather, event, performance, window adapters/ open_meteo, predicthq, asknews, aerodatabox, opensky, cso, hotel_csv, normalize, textdates @@ -448,6 +448,7 @@ which version serves. | GET | `/hotels/{id}/events` | paginated; withdrawn events hidden | | GET | `/hotels/{id}/flight-arrivals` | paginated; one row per source per day | | GET | `/hotels/{id}/recommendations?date=` | active model; optional `room_type_id` | +| GET | `/hotels/{id}/freshness` | last success per kind of data, the latest failure, and whether it's stale | | POST | `/hotels/{id}/train` | job, 202; optional description, window, settings | | GET | `/hotels/{id}/models` | paginated, newest version first | | GET | `/models/{id}` | one version, no artifact |