Feauter llms

.env.ci

@@ -14,12 +14,12 @@ LOG_LEVEL=debug
 
 FLARE_KEY=
 
-DB_CONNECTION=mysql
+DB_CONNECTION=pgsql
 DB_HOST=127.0.0.1
-DB_PORT=3306
-DB_DATABASE=laravel
-DB_USERNAME=root
-DB_PASSWORD=root
+DB_PORT=5432
+DB_DATABASE=testing
+DB_USERNAME=postgres
+DB_PASSWORD=postgres
 
 FPH_ENABLED=true
 CSP_ENABLED=true

.env.example

@@ -57,4 +57,6 @@ CLOUDINARY_CLOUD_NAME=my-cloud-name
 CLOUDINARY_API_KEY=my-api-key
 CLOUDINARY_API_SECRET=my-api-secret
 
-LARAVEL_DEFAULT_FATHOM_SITE_ID=WECFOADW
+LARAVEL_DEFAULT_FATHOM_SITE_ID=WECFOADW
+LITELLM_URL=https://llm.codebar.net
+LITELLM_MASTER_KEY=

.github/workflows/pest_coverage_pull_request.yml

@@ -21,7 +21,7 @@ jobs:
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ env.PHP_VERSION }}
-          extensions: dom, curl, fileinfo, mysql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, xdebug
+          extensions: dom, curl, fileinfo, pgsql, pdo_pgsql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, xdebug
           coverage: xdebug
 
       - name: Prepare the .env file
@@ -45,9 +45,9 @@ jobs:
 
       - name: Setup Database
         run: |
-          sudo systemctl start mysql
-          mysql --user="root" --password="root" -e "CREATE DATABASE laravel character set UTF8mb4 collate utf8mb4_bin;"
-          mysql --user="root" --password="root" -e "CREATE DATABASE logs character set UTF8mb4 collate utf8mb4_bin;"
+          sudo systemctl start postgresql.service
+          sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
+          sudo -u postgres createdb testing
 
       - name: Generate App Key & Run Migrations
         run: |

.github/workflows/pest_pull_request.yml

@@ -19,7 +19,7 @@ jobs:
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ env.PHP_VERSION }}
-          extensions: dom, curl, fileinfo, mysql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick
+          extensions: dom, curl, fileinfo, pgsql, pdo_pgsql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick
 
       - name: Prepare the .env file
         run: cp .env.ci .env
@@ -42,8 +42,9 @@ jobs:
 
       - name: Setup Database
         run: |
-          sudo systemctl start mysql
-          mysql --user="root" --password="root" -e "CREATE DATABASE laravel character set UTF8mb4 collate utf8mb4_bin;"
+          sudo systemctl start postgresql.service
+          sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
+          sudo -u postgres createdb testing
 
       - name: Generate App Key & Run Migrations
         run: |

app/Actions/FetchLlmUsageAction.php

@@ -0,0 +1,60 @@
+<?php
+
+namespace App\Actions;
+
+use Carbon\CarbonImmutable;
+use Illuminate\Http\Client\PendingRequest;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Http;
+
+class FetchLlmUsageAction
+{
+    /**
+     * Fetch the per-model usage aggregates for a single day from the LiteLLM proxy.
+     *
+     * @return Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}>
+     */
+    public function fetchDay(CarbonImmutable $date): Collection
+    {
+        $response = $this->client()->get('/spend/logs', [
+            'start_date' => $date->toDateString(),
+            'end_date' => $date->addDay()->toDateString(),
+            'summarize' => 'false',
+        ])->throw();
+
+        return $this->parseSpendLogs($date, $response->json() ?? []);
+    }
+
+    private function client(): PendingRequest
+    {
+        return Http::baseUrl(config('services.litellm.url'))
+            ->withToken(config('services.litellm.master_key'))
+            ->acceptJson()
+            ->connectTimeout(10)
+            ->timeout(120)
+            ->retry(times: 2, sleepMilliseconds: 1000, throw: false);
+    }
+
+    /**
+     * @return Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}>
+     */
+    private function parseSpendLogs(CarbonImmutable $date, array $logs): Collection
+    {
+        // The end_date bound is inclusive, so the response can contain rows of
+        // the following day — keep only rows that started on the requested day.
+        return collect($logs)
+            ->filter(fn (mixed $log): bool => filled(data_get($log, 'model_group')))
+            ->filter(fn (mixed $log): bool => str_starts_with((string) data_get($log, 'startTime'), $date->toDateString()))
+            ->groupBy(fn (mixed $log): string => data_get($log, 'model_group'))
+            ->map(fn (Collection $rows, string $model): array => [
+                'date' => $date->toDateString(),
+                'model' => $model,
+                'prompt_tokens' => (int) $rows->sum(fn (mixed $log): int => (int) data_get($log, 'prompt_tokens', 0)),
+                'completion_tokens' => (int) $rows->sum(fn (mixed $log): int => (int) data_get($log, 'completion_tokens', 0)),
+                'total_tokens' => (int) $rows->sum(fn (mixed $log): int => (int) data_get($log, 'total_tokens', 0)),
+                'requests' => $rows->count(),
+                'spend' => round($rows->sum(fn (mixed $log): float => (float) data_get($log, 'spend', 0)), 6),
+            ])
+            ->values();
+    }
+}

app/Actions/LlmUsageStatsAction.php

@@ -0,0 +1,134 @@
+<?php
+
+namespace App\Actions;
+
+use App\Models\AiModelDailyUsage;
+use Carbon\CarbonImmutable;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Str;
+
+class LlmUsageStatsAction
+{
+    public const string VERSION_CACHE_KEY = 'llm_usage_version';
+
+    public const string OTHER_MODEL = 'other';
+
+    /**
+     * @return Collection<int, string>
+     */
+    public function models(): Collection
+    {
+        return $this->remember('models', function () {
+            return AiModelDailyUsage::query()
+                ->whereNotNull('ai_model_id')
+                ->distinct()
+                ->orderBy('model')
+                ->pluck('model');
+        });
+    }
+
+    public function hasOtherModels(): bool
+    {
+        return $this->remember('has_other_models', function () {
+            return AiModelDailyUsage::query()->whereNull('ai_model_id')->exists();
+        });
+    }
+
+    /**
+     * @return Collection<int, string>
+     */
+    public function years(): Collection
+    {
+        return $this->remember('years', function () {
+            return AiModelDailyUsage::query()
+                ->orderBy('date')
+                ->pluck('date')
+                ->map(fn (Carbon $date) => $date->format('Y'))
+                ->unique()
+                ->values();
+        });
+    }
+
+    /**
+     * @return Collection<int, array{label: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}>
+     */
+    public function monthlyBreakdown(?string $year, ?string $month, ?string $model): Collection
+    {
+        $suffix = 'breakdown_'.($year ?? 'all').'_'.($month ?? 'all').'_'.($model ?? 'all');
+
+        return $this->remember($suffix, function () use ($year, $month, $model) {
+            return AiModelDailyUsage::query()
+                ->when($model === self::OTHER_MODEL, fn (Builder $query) => $query->whereNull('ai_model_id'))
+                ->when($model && $model !== self::OTHER_MODEL, fn (Builder $query) => $query->where('model', $model))
+                ->when($year, fn (Builder $query) => $query->whereYear('date', $year))
+                ->when($month, fn (Builder $query) => $query->whereMonth('date', $month))
+                ->orderBy('date')
+                ->get()
+                ->groupBy(fn (AiModelDailyUsage $row) => $row->date->format('Y-m'))
+                ->map(fn (Collection $rows, string $label) => [
+                    'label' => $label,
+                    'prompt_tokens' => (int) $rows->sum('prompt_tokens'),
+                    'completion_tokens' => (int) $rows->sum('completion_tokens'),
+                    'total_tokens' => (int) $rows->sum('total_tokens'),
+                    'requests' => (int) $rows->sum('requests'),
+                ])
+                ->values();
+        });
+    }
+
+    /**
+     * @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
+     */
+    public function currentMonthSummary(?string $model = null): array
+    {
+        return $this->summary('month_summary', CarbonImmutable::now()->startOfMonth(), $model);
+    }
+
+    /**
+     * @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
+     */
+    public function currentYearSummary(?string $model = null): array
+    {
+        return $this->summary('year_summary', CarbonImmutable::now()->startOfYear(), $model);
+    }
+
+    /**
+     * @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
+     */
+    public function totalSummary(?string $model = null): array
+    {
+        return $this->summary('total_summary', null, $model);
+    }
+
+    /**
+     * @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
+     */
+    private function summary(string $suffix, ?CarbonImmutable $from, ?string $model = null): array
+    {
+        return $this->remember($suffix.'_'.($model ?? 'all'), function () use ($from, $model) {
+            $rows = AiModelDailyUsage::query()
+                ->when($from, fn (Builder $query) => $query->where('date', '>=', $from))
+                ->when($model === self::OTHER_MODEL, fn (Builder $query) => $query->whereNull('ai_model_id'))
+                ->when($model && $model !== self::OTHER_MODEL, fn (Builder $query) => $query->where('model', $model))
+                ->get();
+
+            return [
+                'prompt_tokens' => (int) $rows->sum('prompt_tokens'),
+                'completion_tokens' => (int) $rows->sum('completion_tokens'),
+                'total_tokens' => (int) $rows->sum('total_tokens'),
+                'requests' => (int) $rows->sum('requests'),
+            ];
+        });
+    }
+
+    private function remember(string $suffix, callable $callback): mixed
+    {
+        $version = Cache::get(self::VERSION_CACHE_KEY, 0);
+        $key = Str::slug("llm_usage_{$version}_{$suffix}", '_');
+
+        return Cache::remember($key, now()->addHour(), $callback);
+    }
+}

app/Actions/StoreLlmUsageAction.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Actions;
+
+use App\Models\AiModel;
+use App\Models\AiModelDailyUsage;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\Cache;
+use Spatie\ResponseCache\Commands\ClearCommand;
+
+class StoreLlmUsageAction
+{
+    /**
+     * @param  Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}>  $rows
+     */
+    public function store(Collection $rows): int
+    {
+        if ($rows->isEmpty()) {
+            return 0;
+        }
+
+        $aiModelIds = AiModel::pluck('id', 'name');
+
+        $rows = $rows->map(fn (array $row) => [
+            ...$row,
+            'ai_model_id' => $aiModelIds->get($row['model']),
+        ]);
+
+        $count = AiModelDailyUsage::upsert(
+            $rows->all(),
+            uniqueBy: ['date', 'model'],
+            update: ['ai_model_id', 'prompt_tokens', 'completion_tokens', 'total_tokens', 'requests', 'spend'],
+        );
+
+        Cache::increment(LlmUsageStatsAction::VERSION_CACHE_KEY);
+
+        Artisan::call(ClearCommand::class);
+
+        return $count;
+    }
+}

app/Actions/ViewDataAction.php

@@ -6,7 +6,6 @@
 use App\Enums\AiModelCategoryEnum;
 use App\Enums\ContactSectionEnum;
 use App\Models\AiModel;
-use App\Models\Configuration;
 use App\Models\Contact;
 use App\Models\News;
 use App\Models\OpenSource;
@@ -19,15 +18,6 @@
 
 class ViewDataAction
 {
-    public function configuration(string $locale): ?Configuration
-    {
-        $key = Str::slug("configuration_{$locale}");
-
-        return Cache::rememberForever($key, function () {
-            return Configuration::first();
-        });
-    }
-
     public function products(string $locale): Collection
     {
         $key = Str::slug("products_published_{$locale}");

app/Console/Commands/FetchLlmAnalyticsCommand.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\FetchLlmUsageJob;
+use Carbon\CarbonImmutable;
+use Carbon\CarbonPeriod;
+use Illuminate\Console\Command;
+
+class FetchLlmAnalyticsCommand extends Command
+{
+    private const string INITIAL_SYNC_START = '2026-01-01';
+
+    protected $signature = 'llm:fetch-analytics
+        {--full : Full sync from the very beginning ('.self::INITIAL_SYNC_START.')}
+        {--from= : Start date (YYYY-MM-DD), defaults to 3 days ago}
+        {--to= : End date (YYYY-MM-DD), defaults to today}';
+
+    protected $description = 'Fetch daily per-model LLM usage from the LiteLLM proxy and store it locally';
+
+    public function handle(): int
+    {
+        $to = $this->date($this->option('to')) ?? CarbonImmutable::today();
+        $from = $this->date($this->option('from'))
+            ?? ($this->option('full') ? $this->date(self::INITIAL_SYNC_START) : $to->subDays(3));
+
+        if ($from->greaterThan($to)) {
+            $this->error('The --from date must not be after the --to date.');
+
+            return self::FAILURE;
+        }
+
+        $days = collect(CarbonPeriod::create($from, $to));
+
+        $days->each(fn (mixed $day) => FetchLlmUsageJob::dispatch(CarbonImmutable::instance($day)));
+
+        $this->info("Dispatched {$days->count()} job(s) for {$from->toDateString()} to {$to->toDateString()}.");
+
+        return self::SUCCESS;
+    }
+
+    private function date(?string $value): ?CarbonImmutable
+    {
+        return $value === null
+            ? null
+            : CarbonImmutable::createFromFormat('!Y-m-d', $value);
+    }
+}