Show page views of people, with a switch to include bots

CLAUDE.md

@@ -480,6 +480,17 @@ section.
   counted once a day with the first page they opened. A request for anything
   else - a script, an API call, `robots.txt`, a path that was not there -
   makes nobody a visitor, so there are never more visitors than page views.
+- **Page views are people's, bots behind a switch - decided by the owner.**
+  The views of programs are counted apart (`bot_views`, in every table), and
+  the editor shows the pages people opened unless the owner switches on
+  **Include bots** - beside the ranges, since it changes the figure, its chart
+  and the page views' cards alike; remembered in their browser, and worked out
+  from the one answer rather than asked for again, and marked "incl. bots"
+  wherever it counts them. Where visitors came from is people's whatever the
+  switch says - a bot comes from nowhere and would read as typed in - and
+  requests and crawlers count every request either way. Days counted before `V2` keep their bots among
+  the people of their pages, referrers and countries; their totals and devices
+  were filled in.
 - **Nothing that names a visitor is kept.** Counts only; the referrer as its
   host; visitors told apart for a day by a hash under a random key per day
   and lambda, held in memory and never written. No cookie, no script in the

README.md

@@ -828,7 +828,14 @@ feature's preview.
 - **What a visit is counted as.** A *hit* is every request answered; a *view*
   is somebody opening a page - a browser's navigation (`Sec-Fetch-Dest:
   document`, a revalidation included) or, from a program that does not send
-  that header, a successful `GET` answered with HTML. A path is counted where
+  that header, a successful `GET` answered with HTML. Of the views, those
+  of programs - crawlers, and whatever names no browser - are counted apart
+  as well (`bot_views`, `botViews` in the answer), everywhere a view is: the
+  editor shows the pages people opened and adds the bots' behind a switch
+  (**Include bots**), off by default and remembered in the browser, worked
+  out from the one answer so it switches at once. The referring sites stay
+  people's either way: a bot names no page it came from and would be counted
+  as typed in. A path is counted where
   it was answered without an error, at most 200 a day, and the rest under
   `(other)`, so a scanner cannot fill a day. A referrer is kept as its host,
   without `www.`, and only where it is another site than the one visited (100

src/Frontend/src/api.ts

@@ -85,8 +85,10 @@ export interface Activation {
 export interface AnalyticsTotals {
   /** Every request answered - pages, the files they load, calls of an API. */
   hits: number;
-  /** The requests that were somebody opening a page. */
+  /** The requests that were somebody opening a page - people and programs alike. */
   views: number;
+  /** Of the views, those of programs - crawlers, and whatever names no browser - rather than people. */
+  botViews: number;
   /** Different visitors, each counted once a day. */
   visitors: number;
   failed: number;
@@ -107,6 +109,8 @@ export interface AnalyticsValue {
   value: string;
   hits: number;
   views: number;
+  /** Of the views, those programs opened rather than people. */
+  botViews: number;
 }
 
 /** How a lambda's visitors reach it, as its owner is shown. */

src/Frontend/src/control/AnalyticsTab.tsx

@@ -10,7 +10,7 @@ import { useEditorT, useLanguage } from '../i18n';
 import { tagOf } from '../i18n/languages';
 import type { Control } from './context';
 import { count, percent } from './format';
-import { Section, Sparkline, pill } from './ui';
+import { Section, Sparkline, Switch, pill } from './ui';
 
 /** The time ranges offered; those longer than the tier shows are offered locked. */
 const RANGES = [7, 30, 90, 365];
@@ -35,6 +35,17 @@ const DEVICES: Record<string, [string, string]> = {
 
 const DEVICE_ORDER = ['desktop', 'mobile', 'tablet', 'bot', 'other'];
 
+/** Whether the owner counts the pages bots opened as well, remembered in their browser. */
+const BOTS = 'lambda-analytics-bots';
+
+function rememberedBots(): boolean {
+  try {
+    return localStorage.getItem(BOTS) === 'yes';
+  } catch {
+    return false;
+  }
+}
+
 /**
  * Who visits the lambda: how many people come each day, which pages they
  * open, which sites send them, from which countries and on what.
@@ -66,6 +77,25 @@ export function AnalyticsTab({ control }: { control: Control }) {
   const [measure, setMeasure] = useState<Measure>('visitors');
   const [switching, setSwitching] = useState(false);
   const [confirming, setConfirming] = useState(false);
+  const [bots, setBots] = useState(rememberedBots);
+
+  const toggleBots = () => {
+    setBots((were) => {
+      try {
+        localStorage.setItem(BOTS, were ? 'no' : 'yes');
+      } catch {
+        // a browser that keeps nothing starts with people only, which is fine
+      }
+
+      return !were;
+    });
+  };
+
+  // the pages people opened, unless the owner asked for those of bots as
+  // well: worked out here rather than asked for again, so the switch answers
+  // at once
+  const people = useMemo(() => state && withoutBots(state), [state]);
+  const shown = bots ? state : people;
 
   /*
    * Read when opened and when the range changes, and again when the page is
@@ -126,7 +156,7 @@ export function AnalyticsTab({ control }: { control: Control }) {
     }
   }
 
-  if (!state) {
+  if (!state || !shown || !people) {
     return (
       <Section title={title}>
         {failure ? (
@@ -150,8 +180,9 @@ export function AnalyticsTab({ control }: { control: Control }) {
   const date = (day: string, options: Intl.DateTimeFormatOptions = { day: 'numeric', month: 'short' }) =>
     new Date(`${day}T00:00:00Z`).toLocaleDateString(tag, { ...options, timeZone: 'UTC' });
 
-  // pages opened where there are any; an app that only answers calls is
-  // described by what it was asked for
+  // pages opened where there are any, by bots included, so the switch never
+  // turns the page into another; an app that only answers calls is described
+  // by what it was asked for
   const by: 'views' | 'hits' = state.totals.views > 0 ? 'views' : 'hits';
   const hidden = state.earliest != null && state.earliest < state.from;
 
@@ -180,15 +211,32 @@ export function AnalyticsTab({ control }: { control: Control }) {
     </div>
   );
 
+  /*
+   * Beside the ranges rather than in the page views' own group, since it
+   * changes the figure at the top and its chart as well. Only where pages
+   * are opened: requests and visitors are what they are either way.
+   */
+  const toolbar = (
+    <>
+      {ranges}
+      {by === 'views' && (
+        <label title={said.botsHint} className="ml-auto inline-flex cursor-pointer items-center gap-2 text-[13px] text-slate-600 dark:text-slate-400">
+          <span id="analytics-bots">{said.bots}</span>
+          <Switch on={bots} onToggle={toggleBots} labelledBy="analytics-bots" />
+        </label>
+      )}
+    </>
+  );
+
   return (
-    <Section title={title} hint={said.hint} pills={ranges}>
+    <Section title={title} hint={said.hint} pills={toolbar}>
       {hidden && lambda.tier !== 'Premium' && (
         <Nudge since={date(state.earliest!, { day: 'numeric', month: 'long', year: 'numeric' })} address={absoluteAddress(lambda.address)} />
       )}
 
-      <Figures state={state} measure={measure} onPick={setMeasure} dark={dark} />
+      <Figures state={shown} measure={measure} onPick={setMeasure} dark={dark} bots={bots} />
 
-      <Columns state={state} measure={measure} dark={dark} date={date} />
+      <Columns state={shown} measure={measure} dark={dark} date={date} />
 
       {/*
         * In two groups by what their numbers count, since the two can be far
@@ -197,31 +245,32 @@ export function AnalyticsTab({ control }: { control: Control }) {
         * people open is described by its requests alone.
         */}
       {by === 'views' && (
-        <Group title={said.views} hint={said.viewsHint}>
-          <Card title={said.pages} hint={said.pagesHint}>
+        <Group title={said.views} hint={said.viewsHint} badge={bots ? said.withBots : undefined}>
+          <Card title={said.pages} hint={bots ? said.pagesHintBots : said.pagesHint}>
             <Ranked
-              values={(state.pages ?? []).filter((p) => p.views > 0)}
+              values={(shown.pages ?? []).filter((p) => p.views > 0)}
               by="views"
               mono
               name={(value) => (value === '(other)' ? said.other : value)}
             />
           </Card>
 
+          {/* people's, whatever the switch says: a bot comes from nowhere, and would be counted as typed in */}
           <Card title={said.sources} hint={said.sourcesHint}>
             <Ranked
-              values={(state.referrers ?? []).filter((r) => r.views > 0)}
+              values={(people.referrers ?? []).filter((r) => r.views > 0)}
               by="views"
               name={(value) => (value === '(other)' ? said.other : value)}
               icon={<IconGlobe className="h-3.5 w-3.5 shrink-0 text-slate-400" />}
-              rest={{ label: said.direct, value: Math.max(0, state.totals.views - sum(state.referrers, 'views')) }}
+              rest={{ label: said.direct, value: Math.max(0, people.totals!.views - sum(people.referrers, 'views')) }}
             />
           </Card>
 
           <Card title={said.countries} hint={said.countriesHint}>
-            <Countries values={(state.countries ?? []).filter((c) => c.views > 0)} by="views" tag={tag} />
+            <Countries values={(shown.countries ?? []).filter((c) => c.views > 0)} by="views" tag={tag} />
           </Card>
 
-          <Agents devices={state.devices ?? []} browsers={state.browsers ?? []} systems={state.systems ?? []} by="views" dark={dark} />
+          <Agents devices={shown.devices ?? []} browsers={shown.browsers ?? []} systems={shown.systems ?? []} by="views" dark={dark} />
         </Group>
       )}
 
@@ -365,7 +414,13 @@ function Nudge({ since, address }: { since: string; address: string }) {
  * range, how it changed against the days before where those are shown, and
  * its days in small.
  */
-function Figures({ state, measure, onPick, dark }: { state: LambdaAnalytics; measure: Measure; onPick: (m: Measure) => void; dark: boolean }) {
+function Figures({ state, measure, onPick, dark, bots }: {
+  state: LambdaAnalytics;
+  measure: Measure;
+  onPick: (m: Measure) => void;
+  dark: boolean;
+  bots: boolean;
+}) {
   const said = useEditorT().analytics;
 
   const figures: { id: Measure; label: string; hint: string }[] = [
@@ -397,7 +452,11 @@ function Figures({ state, measure, onPick, dark }: { state: LambdaAnalytics; mea
               className="absolute inset-x-0 top-0 h-0.5"
               style={{ background: active ? ACCENT[dark ? 1 : 0] : 'transparent' }}
             />
-            <div className={`text-[13px] ${active ? 'font-medium text-slate-900 dark:text-slate-100' : 'text-slate-500'}`}>{figure.label}</div>
+            <div className={`flex flex-wrap items-center gap-x-1.5 text-[13px] ${active ? 'font-medium text-slate-900 dark:text-slate-100' : 'text-slate-500'}`}>
+              {figure.label}
+              {/* so a figure read on its own, or in a screenshot, says what it counts */}
+              {figure.id === 'views' && bots && <Badge>{said.withBots}</Badge>}
+            </div>
             <div className="mt-1 text-xl font-semibold tabular-nums tracking-tight sm:text-3xl">{count(value)}</div>
             <Change now={value} before={before} days={state.days} />
             <div className="mt-3 hidden sm:block">
@@ -631,16 +690,24 @@ function useWidth(initial: number) {
 }
 
 /** The cards whose numbers count the same thing, under the name of the figure they count. */
-function Group({ title, hint, children }: { title: string; hint: string; children: ReactNode }) {
+function Group({ title, hint, badge, children }: { title: string; hint: string; badge?: string; children: ReactNode }) {
   return (
     <section className="mt-8">
-      <h2 className="text-base font-semibold tracking-tight">{title}</h2>
+      <h2 className="flex flex-wrap items-center gap-x-2 text-base font-semibold tracking-tight">
+        {title}
+        {badge && <Badge>{badge}</Badge>}
+      </h2>
       <p className="mt-0.5 text-xs text-slate-500">{hint}</p>
       <div className="mt-3 grid gap-6 lg:grid-cols-2">{children}</div>
     </section>
   );
 }
 
+/** A word beside a heading that says what its numbers count besides the usual. */
+function Badge({ children }: { children: ReactNode }) {
+  return <span className="rounded-sm bg-slate-100 px-1 text-[11px] font-normal tracking-normal text-slate-500 dark:bg-ink-800 dark:text-slate-400">{children}</span>;
+}
+
 /** A card of a group. */
 function Card({ title, hint, children }: { title: string; hint?: string; children: ReactNode }) {
   return (
@@ -868,5 +935,33 @@ function niceCeiling(value: number) {
   return 10 * power;
 }
 
+/**
+ * The pages people opened: what the analytics are without the views of bots,
+ * everywhere a view is counted - the totals, the days and every breakdown.
+ */
+function withoutBots(state: LambdaAnalytics): LambdaAnalytics {
+  const people = <T extends { views: number; botViews: number }>(counted: T): T => ({
+    ...counted,
+    views: Math.max(0, counted.views - (counted.botViews ?? 0)),
+    botViews: 0,
+  });
+
+  const list = (values: AnalyticsValue[] | null) => values?.map(people) ?? null;
+
+  return {
+    ...state,
+    totals: state.totals && people(state.totals),
+    previous: state.previous && people(state.previous),
+    daily: state.daily?.map((day) => ({ ...day, totals: people(day.totals) })) ?? null,
+    pages: list(state.pages),
+    referrers: list(state.referrers),
+    countries: list(state.countries),
+    devices: list(state.devices),
+    browsers: list(state.browsers),
+    systems: list(state.systems),
+    crawlers: list(state.crawlers),
+  };
+}
+
 const sum = (values: AnalyticsValue[] | null, by: keyof Pick<AnalyticsTotals, 'views' | 'hits'>) =>
   (values ?? []).reduce((total, v) => total + v[by], 0);

src/Frontend/src/locales/de/editor/analytics.tsx

@@ -27,6 +27,9 @@ export const analytics: EditorMessages['analytics'] = {
   visitorsHint: 'Verschiedene Personen, die eine Seite geöffnet haben, jede einmal pro Tag gezählt. Suchmaschinen und andere Programme sind keine Besucher.',
   views: 'Seitenaufrufe',
   viewsHint: 'Jedes Mal, wenn jemand eine Seite geöffnet hat.',
+  bots: 'Bots einbeziehen',
+  botsHint: 'Zählt auch die Seiten, die Suchmaschinen, KI-Crawler und andere Programme geöffnet haben. Besucher sind in jedem Fall Menschen.',
+  withBots: 'inkl. Bots',
   hits: 'Requests',
   hitsHint: 'Alles, was bei Ihrer App angefragt wurde: ihre Seiten sowie die Bilder und Skripte, die sie laden.',
   versus: (days) => `gegenüber den ${days} Tagen davor`,
@@ -53,6 +56,7 @@ export const analytics: EditorMessages['analytics'] = {
 
   pages: 'Seiten',
   pagesHint: 'Die Seiten, die geöffnet wurden – die am häufigsten geöffneten zuerst.',
+  pagesHintBots: 'Die Seiten, die Menschen und Bots geöffnet haben – die am häufigsten geöffneten zuerst.',
   requested: 'Am häufigsten angefragt',
   requestedHint: 'Alles, was bei Ihrer App angefragt wurde, nach Adresse – sie hat keine Seiten, die Personen öffnen.',
   sources: 'Woher die Besucher kommen',

src/Frontend/src/locales/de/guide.tsx

@@ -511,7 +511,9 @@ export const guide: Messages['guide'] = {
       Öffnen Sie im Kontrollzentrum {k.b('Analyse')} und schalten Sie sie ein, um zu sehen, wer Ihre App besucht: wie viele
       Personen jeden Tag kommen, welche Seiten sie öffnen, welche Websites sie schicken – eine Suchmaschine, ein Beitrag, ein
       Newsletter –, aus welchen Ländern, auf welchen Geräten und in welchen Browsern, und welche Suchmaschinen und
-      KI-Assistenten ihre Seiten lesen. Wählen Sie oben eine Zahl, um sie Tag für Tag zu sehen.
+      KI-Assistenten ihre Seiten lesen. Wählen Sie oben eine Zahl, um sie Tag für Tag zu sehen. Seitenaufrufe sind die Seiten,
+      die Personen geöffnet haben; schalten Sie {k.b('Bots einbeziehen')} ein, um auch die von Suchmaschinen, KI-Crawlern und
+      anderen Programmen zu zählen.
     </>
   )),
   visitors2: ((k) => (

src/Frontend/src/locales/en/editor/analytics.tsx

@@ -33,6 +33,11 @@ export const analytics = {
   visitorsHint: 'Different people who opened a page, each counted once a day. Search engines and other programs are not visitors.',
   views: 'Page views',
   viewsHint: 'Every time somebody opened a page.',
+  /** The switch beside the time ranges: whether the page views count the pages bots opened as well. Off, they count people only. */
+  bots: 'Include bots',
+  botsHint: 'Count the pages that search engines, AI crawlers and other programs opened as well. Visitors are people either way.',
+  /** Beside "Page views" at the top while bots are included. */
+  withBots: 'incl. bots',
   hits: 'Requests',
   hitsHint: 'Everything your app was asked for: its pages, and the pictures and scripts they load.',
   /** Under a figure: how it changed against the days before the ones shown. */
@@ -62,6 +67,8 @@ export const analytics = {
 
   pages: 'Pages',
   pagesHint: 'The pages people opened, the most opened first.',
+  /** The same, while the bots are included. */
+  pagesHintBots: 'The pages people and bots opened, the most opened first.',
   requested: 'Most requested',
   requestedHint: 'Everything your app was asked for, by address - it has no pages people open.',
   sources: 'Where visitors come from',

src/Frontend/src/locales/en/guide.tsx

@@ -507,7 +507,8 @@ export const guide = {
       Open {k.b('Analytics')} in the control center and switch it on to see who visits your app: how many people
       come each day, which pages they open, which sites send them - a search engine, a post, a newsletter - from
       which countries, on which devices and browsers, and which search engines and AI assistants read its pages.
-      Pick a figure at the top to see it day by day.
+      Pick a figure at the top to see it day by day. Page views are the pages people opened; switch on {k.b('Include bots')}
+      to count those of search engines, AI crawlers and other programs as well.
     </>
   )) as Text,
   visitors2: ((k) => (

src/Frontend/src/locales/es/editor/analytics.tsx

@@ -27,6 +27,9 @@ export const analytics: EditorMessages['analytics'] = {
   visitorsHint: 'Personas distintas que abrieron una página, cada una contada una vez al día. Los buscadores y otros programas no son visitantes.',
   views: 'Visualizaciones de página',
   viewsHint: 'Cada vez que alguien abrió una página.',
+  bots: 'Incluir bots',
+  botsHint: 'Cuenta también las páginas que abrieron buscadores, rastreadores de IA y otros programas. Los visitantes son siempre personas.',
+  withBots: 'con bots',
   hits: 'Peticiones',
   hitsHint: 'Todo lo que se pidió a tu app: sus páginas y también las imágenes y los scripts que cargan.',
   versus: (days) => `frente a los ${days} días anteriores`,
@@ -53,6 +56,7 @@ export const analytics: EditorMessages['analytics'] = {
 
   pages: 'Páginas',
   pagesHint: 'Las páginas que abrió la gente, las más abiertas primero.',
+  pagesHintBots: 'Las páginas que abrieron la gente y los bots, las más abiertas primero.',
   requested: 'Lo más pedido',
   requestedHint: 'Todo lo que se pidió a tu app, por dirección: no tiene páginas que la gente abra.',
   sources: 'De dónde vienen los visitantes',

src/Frontend/src/locales/es/guide.tsx

@@ -501,7 +501,8 @@ export const guide: Messages['guide'] = {
       Abre {k.b('Analítica')} en el centro de control y actívala para ver quién visita tu app: cuánta gente llega cada
       día, qué páginas abre, qué sitios la envían (un buscador, una publicación, un boletín), desde qué países, en qué
       dispositivos y navegadores, y qué buscadores y asistentes de IA leen sus páginas. Elige una cifra arriba para
-      verla día a día.
+      verla día a día. Las visitas a páginas son las páginas que la gente abrió; activa {k.b('Incluir bots')} para
+      contar también las de buscadores, rastreadores de IA y otros programas.
     </>
   ),
   visitors2: (k) => (