diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index de59ec94..48835393 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -147,6 +147,31 @@ Reports will stick around until deleted, so you are free to compare reports to s You may generate a report through the Control Panel, or by running `php please seo-pro:generate-report`. +### Storage + +By default, reports are stored as YAML files in the `storage/statamic/seopro/reports` directory. You can change this in the config: + +```php +// config/statamic/seo-pro.php + +'reports' => [ + 'driver' => 'file', + 'directory' => storage_path('statamic/seopro/reports'), +], +``` + +Where the filesystem is not shared between processes or does not survive a deploy (containers, serverless platforms, autoscaled instances), a report written by one process is never seen by the next request. In that case you may store reports in the database by changing the driver: + +```php +// config/statamic/seo-pro.php + +'reports' => [ + 'driver' => 'database', +], +``` + +Then run `php please seo-pro:database-reports` to publish the migration and import existing reports. + ### Queuing Reports Depending on the size of your site, generating a report may take a while. To prevent request timeouts, you may enable queues, and the reports will be truly queued in the background. diff --git a/config/seo-pro.php b/config/seo-pro.php index 6415c6ad..5722e83d 100644 --- a/config/seo-pro.php +++ b/config/seo-pro.php @@ -50,6 +50,8 @@ ], 'reports' => [ + 'driver' => 'file', + 'directory' => storage_path('statamic/seopro/reports'), 'keep_recent' => 'all', 'queue_chunk_size' => 1000, 'title_length' => [ diff --git a/src/Commands/PublishReportMigrations.php b/src/Commands/PublishReportMigrations.php new file mode 100644 index 00000000..15e62ed5 --- /dev/null +++ b/src/Commands/PublishReportMigrations.php @@ -0,0 +1,103 @@ +publishMigration() + ->runMigration() + ->importReports(); + } + + private function publishMigration(): self + { + $name = 'create_seo_pro_reports_tables.php'; + + $existingMigration = collect(File::allFiles(database_path('migrations'))) + ->map->getFilename() + ->filter(fn (string $filename) => Str::contains($filename, $name)) + ->first(); + + if ($existingMigration) { + $this->components->info("Migration [database/migrations/{$existingMigration}] already exists."); + + return $this; + } + + $filename = date('Y_m_d_His').'_'.$name; + + $contents = File::get(__DIR__.'/stubs/'.$name); + + File::put(database_path('migrations/'.$filename), $contents); + + $this->components->info("Migration [database/migrations/{$filename}] published successfully."); + + return $this; + } + + private function runMigration(): self + { + Artisan::call('migrate', ['--force' => true], $this->output); + + $this->newLine(); + + return $this; + } + + private function importReports(): self + { + if (! confirm('Would you like to import existing reports?')) { + return $this; + } + + $files = app(FileReportRepository::class); + $database = app(EloquentReportRepository::class); + + $reports = $files->all(); + + if ($reports->isEmpty()) { + $this->components->info('There are no reports to import.'); + + return $this; + } + + $progress = progress(label: 'Importing reports', steps: $reports->count()); + + $progress->start(); + + $reports->each(function (Report $report) use ($files, $database, $progress) { + $database->save($report); + + $files->pages($report)->each(fn ($page) => $database->savePage($page)); + + $progress->advance(); + }); + + $progress->finish(); + + $this->components->info('Reports imported successfully.'); + + return $this; + } +} diff --git a/src/Commands/stubs/create_seo_pro_reports_tables.php b/src/Commands/stubs/create_seo_pro_reports_tables.php new file mode 100644 index 00000000..675323d0 --- /dev/null +++ b/src/Commands/stubs/create_seo_pro_reports_tables.php @@ -0,0 +1,56 @@ +id(); + $table->dateTime('date'); + $table->string('status'); + $table->float('score')->nullable(); + $table->integer('pages_crawled')->nullable(); + $table->integer('pages_actionable')->nullable(); + $table->json('results')->nullable(); + $table->timestamps(); + }); + + Schema::create('seo_pro_report_pages', function (Blueprint $table) { + $table->id(); + $table->unsignedBigInteger('report_id'); + $table->string('page_id'); + $table->json('data'); + $table->json('results')->nullable(); + $table->timestamps(); + + $table->unique(['report_id', 'page_id']); + }); + + Schema::create('seo_pro_report_chunks', function (Blueprint $table) { + $table->id(); + $table->unsignedBigInteger('report_id'); + $table->integer('chunk_id'); + $table->json('content_ids'); + $table->timestamps(); + + $table->index('report_id'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('seo_pro_report_chunks'); + Schema::dropIfExists('seo_pro_report_pages'); + Schema::dropIfExists('seo_pro_reports'); + } +}; diff --git a/src/Reporting/Chunk.php b/src/Reporting/Chunk.php index 6424d34e..2c6a14ec 100644 --- a/src/Reporting/Chunk.php +++ b/src/Reporting/Chunk.php @@ -4,8 +4,6 @@ use Illuminate\Support\Facades\Cache; use Statamic\Facades\Data; -use Statamic\Facades\File; -use Statamic\Facades\YAML; use Statamic\SeoPro\Cascade; use Statamic\SeoPro\GetsSectionDefaults; use Statamic\SeoPro\SiteDefaults\SiteDefaults; @@ -25,21 +23,24 @@ public function __construct($id, $contentIds, Report $report) $this->report = $report; } - protected function folderPath() + public function id() { - return $this->report->chunksFolder()."/{$this->id}"; + return $this->id; } - protected function yamlPath() + public function contentIds() { - return $this->folderPath().'/chunk.yaml'; + return $this->contentIds; + } + + public function report() + { + return $this->report; } public function save() { - File::put($this->yamlPath(), YAML::dump([ - 'ids' => $this->contentIds, - ])); + $this->repository()->saveChunk($this); return $this; } @@ -61,7 +62,7 @@ protected function generate($ids) $this->createPage($content[$id] ?? Data::find($id))?->save(); } - File::delete($this->folderPath()); + $this->repository()->deleteChunk($this); if ($this->wasLastChunk()) { $this->report->finalize(); @@ -70,7 +71,12 @@ protected function generate($ids) protected function wasLastChunk() { - return File::getFolders($this->report->chunksFolder())->isEmpty(); + return ! $this->repository()->hasChunks($this->report); + } + + protected function repository() + { + return app(ReportRepository::class); } protected function createPage($content) diff --git a/src/Reporting/Eloquent/ReportChunkModel.php b/src/Reporting/Eloquent/ReportChunkModel.php new file mode 100644 index 00000000..8acabcfa --- /dev/null +++ b/src/Reporting/Eloquent/ReportChunkModel.php @@ -0,0 +1,19 @@ + 'json', + ]; + } +} diff --git a/src/Reporting/Eloquent/ReportModel.php b/src/Reporting/Eloquent/ReportModel.php new file mode 100644 index 00000000..aa3712c7 --- /dev/null +++ b/src/Reporting/Eloquent/ReportModel.php @@ -0,0 +1,23 @@ + 'datetime', + 'score' => 'float', + 'pages_crawled' => 'integer', + 'pages_actionable' => 'integer', + 'results' => 'json', + ]; + } +} diff --git a/src/Reporting/Eloquent/ReportPageModel.php b/src/Reporting/Eloquent/ReportPageModel.php new file mode 100644 index 00000000..62adb0f4 --- /dev/null +++ b/src/Reporting/Eloquent/ReportPageModel.php @@ -0,0 +1,20 @@ + 'json', + 'results' => 'json', + ]; + } +} diff --git a/src/Reporting/Eloquent/ReportRepository.php b/src/Reporting/Eloquent/ReportRepository.php new file mode 100644 index 00000000..fcc4bbf7 --- /dev/null +++ b/src/Reporting/Eloquent/ReportRepository.php @@ -0,0 +1,129 @@ +orderByDesc('id') + ->get() + ->map(fn ($model) => static::fromModel($model)); + } + + public function find($id): ?Report + { + if (! is_numeric($id) || ! $model = ReportModel::find((int) $id)) { + return null; + } + + return static::fromModel($model); + } + + public function latest(): ?Report + { + $model = ReportModel::query()->orderByDesc('id')->first(); + + return $model ? static::fromModel($model) : null; + } + + public function nextId(): int + { + return (int) ReportModel::max('id') + 1; + } + + public function save(Report $report): void + { + $raw = $report->raw(); + + ReportModel::updateOrCreate(['id' => $report->id()], [ + 'date' => $raw['date'], + 'status' => $raw['status'], + 'score' => $raw['score'], + 'pages_crawled' => $raw['pages_crawled'], + 'pages_actionable' => $raw['pages_actionable'], + 'results' => $raw['results'], + ]); + } + + public function delete(Report $report): void + { + DB::transaction(function () use ($report) { + ReportPageModel::query()->where('report_id', $report->id())->delete(); + ReportChunkModel::query()->where('report_id', $report->id())->delete(); + ReportModel::query()->where('id', $report->id())->delete(); + }); + } + + public function pages(Report $report): Collection + { + return ReportPageModel::query() + ->where('report_id', $report->id()) + ->get() + ->map(fn ($model) => (new Page($model->page_id, $model->data, $report))->setResults($model->results)); + } + + public function savePage(Page $page): void + { + // One query per page: a report saves every page twice (the chunk, then finalize()). + ReportPageModel::upsert([[ + 'report_id' => $page->report()->id(), + 'page_id' => $page->id(), + 'data' => json_encode($page->data()), + 'results' => json_encode($page->results()), + ]], ['report_id', 'page_id'], ['data', 'results']); + } + + public function saveChunk(Chunk $chunk): void + { + ReportChunkModel::updateOrCreate([ + 'report_id' => $chunk->report()->id(), + 'chunk_id' => $chunk->id(), + ], [ + 'content_ids' => $chunk->contentIds(), + ]); + } + + public function deleteChunk(Chunk $chunk): void + { + ReportChunkModel::query() + ->where('report_id', $chunk->report()->id()) + ->where('chunk_id', $chunk->id()) + ->delete(); + } + + public function hasChunks(Report $report): bool + { + return ReportChunkModel::query()->where('report_id', $report->id())->exists(); + } + + public function deleteChunks(Report $report): void + { + ReportChunkModel::query()->where('report_id', $report->id())->delete(); + } + + public static function bindings(): array + { + return []; + } + + public static function fromModel(ReportModel $model): Report + { + return Report::create($model->id)->hydrate([ + 'date' => $model->date->timestamp, + 'status' => $model->status, + 'score' => $model->score, + 'pages_crawled' => $model->pages_crawled, + 'pages_actionable' => $model->pages_actionable, + 'results' => $model->results, + ]); + } +} diff --git a/src/Reporting/File/ReportRepository.php b/src/Reporting/File/ReportRepository.php new file mode 100644 index 00000000..c011dbbe --- /dev/null +++ b/src/Reporting/File/ReportRepository.php @@ -0,0 +1,145 @@ +prepareDirectory()) + ->map(fn ($path) => (int) pathinfo($path)['filename']) + ->map(fn ($id) => $this->find($id)) + ->filter() + ->sortByDesc + ->id() + ->values(); + } + + public function find($id): ?Report + { + if (! File::exists($path = $this->reportPath($id))) { + return null; + } + + return Report::create($id)->hydrate(YAML::parse(File::get($path))); + } + + public function latest(): ?Report + { + return $this->all()->first(); + } + + public function nextId(): int + { + return (int) Folder::getFolders($this->prepareDirectory()) + ->map(fn ($path) => (int) pathinfo($path)['filename']) + ->max() + 1; + } + + public function save(Report $report): void + { + File::put($this->reportPath($report->id()), YAML::dump($report->raw())); + } + + public function delete(Report $report): void + { + File::delete($this->reportFolder($report->id())); + } + + public function pages(Report $report): Collection + { + return Folder::getFilesRecursively($this->reportFolder($report->id()).'/pages') + ->map(fn ($file) => YAML::parse(File::get($file))) + ->map(fn ($yaml) => (new Page($yaml['id'], $yaml['data'], $report))->setResults($yaml['results'])); + } + + public function savePage(Page $page): void + { + File::put($this->pagePath($page), YAML::dump([ + 'id' => $page->id(), + 'data' => $page->data(), + 'results' => $page->results(), + ])); + } + + public function saveChunk(Chunk $chunk): void + { + File::put($this->chunkFolder($chunk).'/chunk.yaml', YAML::dump([ + 'ids' => $chunk->contentIds(), + ])); + } + + public function deleteChunk(Chunk $chunk): void + { + File::delete($this->chunkFolder($chunk)); + } + + public function hasChunks(Report $report): bool + { + return File::exists($dir = $this->chunksFolder($report->id())) + && Folder::getFolders($dir)->isNotEmpty(); + } + + public function deleteChunks(Report $report): void + { + File::delete($this->chunksFolder($report->id())); + } + + public static function bindings(): array + { + return []; + } + + private function directory(): string + { + return config('statamic.seo-pro.reports.directory', storage_path('statamic/seopro/reports')); + } + + private function prepareDirectory(): string + { + if (! File::exists($directory = $this->directory())) { + File::makeDirectory($directory, 0755, true); + } + + return $directory; + } + + private function reportFolder($id): string + { + return $this->directory().'/'.$id; + } + + private function reportPath($id): string + { + return $this->reportFolder($id).'/report.yaml'; + } + + private function chunksFolder($id): string + { + return $this->reportFolder($id).'/chunks'; + } + + private function chunkFolder(Chunk $chunk): string + { + return $this->chunksFolder($chunk->report()->id()).'/'.$chunk->id(); + } + + private function pagePath(Page $page): string + { + $key = md5($page->id()); + $parts = array_slice(str_split($key, 2), 0, 2); + + return $this->reportFolder($page->report()->id()).vsprintf('/pages/%s/%s.yaml', [ + implode('/', $parts), $key, + ]); + } +} diff --git a/src/Reporting/Page.php b/src/Reporting/Page.php index b811ddb1..b6d3eb09 100644 --- a/src/Reporting/Page.php +++ b/src/Reporting/Page.php @@ -4,8 +4,6 @@ use Illuminate\Contracts\Support\Arrayable; use Statamic\Facades\Data; -use Statamic\Facades\File; -use Statamic\Facades\YAML; use Statamic\Support\Arr; use Statamic\Support\Str; @@ -40,6 +38,11 @@ public function results() return $this->results; } + public function data() + { + return $this->data; + } + public function validate() { $results = []; @@ -140,23 +143,12 @@ public function id() public function save() { - $data = [ - 'id' => $this->id, - 'data' => $this->data, - 'results' => $this->results, - ]; - - File::put($this->path(), YAML::dump($data)); + $this->repository()->savePage($this); } - protected function path() + protected function repository() { - $key = md5($this->id); - $parts = array_slice(str_split($key, 2), 0, 2); - - return storage_path(vsprintf('statamic/seopro/reports/%s/pages/%s/%s.yaml', [ - $this->report->id(), implode('/', $parts), $key, - ])); + return app(ReportRepository::class); } public function model() diff --git a/src/Reporting/Report.php b/src/Reporting/Report.php index a781ad49..b36ecc62 100644 --- a/src/Reporting/Report.php +++ b/src/Reporting/Report.php @@ -8,10 +8,7 @@ use Illuminate\Support\Facades\Artisan; use Illuminate\Support\Facades\Cache; use Statamic\Facades\Entry; -use Statamic\Facades\File; -use Statamic\Facades\Folder; use Statamic\Facades\Term; -use Statamic\Facades\YAML; class Report implements Arrayable, Jsonable { @@ -57,11 +54,7 @@ public function setId($id) public static function nextId() { - if (! $latest = static::latest()) { - return 1; - } - - return $latest->id() + 1; + return app(ReportRepository::class)->nextId(); } public function id() @@ -112,15 +105,14 @@ public function generate() protected function hasRemainingChunks() { - return File::exists($this->chunksFolder()) - && File::getFolders($this->chunksFolder())->isNotEmpty(); + return $this->repository()->hasChunks($this); } protected function chunks() { $chunks = $this ->allContent() - ->chunk(config('statamic.seo-pro.reports.queue_chunk_size')) + ->chunk(config('statamic.seo-pro.reports.queue_chunk_size', 1000)) ->map(function ($chunk, $id) { return app() ->makeWith(Chunk::class, [ @@ -139,9 +131,7 @@ protected function chunks() public function finalize() { - if (File::exists($dir = $this->chunksFolder())) { - File::delete($dir); - } + $this->repository()->deleteChunks($this); $this ->withPages() @@ -284,13 +274,7 @@ public function withPages($preferCache = false) return $this; } - $dir = storage_path(sprintf('statamic/seopro/reports/%s/pages', $this->id)); - - $files = Folder::getFilesRecursively($dir); - - $this->pages = collect($files) - ->map(fn ($file) => YAML::parse(File::get($file))) - ->map(fn ($yaml) => (new Page($yaml['id'], $yaml['data'], $this))->setResults($yaml['results'])) + $this->pages = $this->repository()->pages($this) ->sortBy ->url() ->values(); @@ -326,24 +310,12 @@ public function generateIfNecessary(): self public static function all() { - $folders = collect(Folder::getFolders(static::preparePath())); - - if ($folders->isEmpty()) { - return $folders; - } - - return $folders - ->map(fn ($path) => (int) pathinfo($path)['filename']) - ->map(fn ($id) => static::find($id)) - ->filter() - ->sortByDesc - ->id() - ->values(); + return app(ReportRepository::class)->all(); } public static function latest() { - return static::all()->first(); + return app(ReportRepository::class)->latest(); } public static function latestGenerated() @@ -353,13 +325,7 @@ public static function latestGenerated() public static function find($id) { - $instance = static::create($id); - - if (! $instance->exists()) { - return; - } - - return $instance->load(); + return app(ReportRepository::class)->find($id); } public function fresh() @@ -369,14 +335,11 @@ public function fresh() public function save() { - File::put($this->path(), YAML::dump($this->raw = [ - 'date' => $this->date ?? now()->timestamp, - 'status' => $this->status(), - 'score' => $this->score(), - 'pages_crawled' => $this->pagesCrawled(), - 'pages_actionable' => $this->pagesActionable(), - 'results' => $this->results, - ])); + $this->date ??= now()->timestamp; + + $this->raw = $this->raw(); + + $this->repository()->save($this); static::deleteOldReports(); @@ -385,34 +348,26 @@ public function save() public function delete() { - File::delete($this->parentFolder()); + $this->repository()->delete($this); return $this->clearCaches(); } - public function parentFolder() + public function raw() { - return storage_path('statamic/seopro/reports/'.$this->id); - } - - public function chunksFolder() - { - return storage_path('statamic/seopro/reports/'.$this->id.'/chunks'); - } - - public function path() - { - return storage_path('statamic/seopro/reports/'.$this->id.'/report.yaml'); - } - - public function exists() - { - return File::exists($this->path()); + return [ + 'date' => $this->date ?? now()->timestamp, + 'status' => $this->status(), + 'score' => $this->score(), + 'pages_crawled' => $this->pagesCrawled(), + 'pages_actionable' => $this->pagesActionable(), + 'results' => $this->results, + ]; } - public function load() + public function hydrate(array $raw) { - $this->raw = YAML::parse(File::get($this->path())); + $this->raw = $raw; $this->date = $this->raw['date']; $this->results = $this->raw['results']; @@ -499,15 +454,9 @@ protected function generateScore() return $this; } - public static function preparePath($path = null) + protected function repository() { - $path = collect([storage_path('statamic/seopro/reports'), $path])->filter()->implode('/'); - - if (! File::exists($path)) { - File::makeDirectory($path, 0755, true); - } - - return $path; + return app(ReportRepository::class); } protected function allContent() @@ -562,16 +511,7 @@ public static function deleteOldReports() return false; } - $reports = collect(app('files')->directories(storage_path('statamic/seopro/reports'))); - - $keepReports = $reports - ->keyBy(fn ($path) => pathinfo($path)['basename']) - ->sortKeysDesc() - ->take($keep); - - $reports - ->reject(fn ($path) => $keepReports->contains($path)) - ->each(fn ($path) => app('files')->deleteDirectory($path)); + static::all()->slice($keep)->each->delete(); return true; } diff --git a/src/Reporting/ReportRepository.php b/src/Reporting/ReportRepository.php new file mode 100644 index 00000000..4b04f482 --- /dev/null +++ b/src/Reporting/ReportRepository.php @@ -0,0 +1,34 @@ +bootAddonSubscriber() ->bootAddonGlidePresets() ->bootRedirects() + ->bootReports() ->bootRouteBindings() ->bootGit() ->bootAddonScheduledCommands() @@ -234,6 +236,17 @@ protected function bootRedirects() return $this; } + protected function bootReports() + { + Statamic::repository(ReportRepository::class, Reporting\File\ReportRepository::class); + + if (config('statamic.seo-pro.reports.driver') === 'database') { + Statamic::repository(ReportRepository::class, Reporting\Eloquent\ReportRepository::class); + } + + return $this; + } + protected function bootRouteBindings(): static { Route::bind('error', function ($id, $route = null) { diff --git a/tests/Reporting/Eloquent/GenerateReportTest.php b/tests/Reporting/Eloquent/GenerateReportTest.php new file mode 100644 index 00000000..d59c3b9e --- /dev/null +++ b/tests/Reporting/Eloquent/GenerateReportTest.php @@ -0,0 +1,76 @@ +each->delete(); + Term::all()->each->delete(); + + if ($this->files->exists($path = storage_path('statamic/seopro/reports'))) { + $this->files->deleteDirectory($path); + } + } + + protected function getEnvironmentSetUp($app) + { + parent::getEnvironmentSetUp($app); + + $app['config']->set('statamic.seo-pro.reports.driver', 'database'); + } + + protected function defineDatabaseMigrations() + { + $this->loadMigrationsFrom(__DIR__.'/../../../src/Commands/stubs'); + } + + #[Test] + public function it_generates_a_report_into_the_database() + { + $this->generateEntries(5)->generateTerms(5); + + Report::create()->save()->generate(); + + $this->assertDatabaseHas('seo_pro_reports', ['id' => 1, 'status' => 'fail', 'pages_crawled' => 10, 'pages_actionable' => 10]); + $this->assertDatabaseCount('seo_pro_report_pages', 10); + $this->assertDatabaseCount('seo_pro_report_chunks', 0); + + $report = Report::find(1); + $this->assertSame(86.0, $report->score()); + $this->assertSame('fail', $report->status()); + $this->assertTrue($report->isGenerated()); + $this->assertCount(10, $report->withPages()->pages()); + $this->assertSame('fail', $report->pages()->first()->status()); + $this->assertFileDoesNotExist(storage_path('statamic/seopro/reports')); + } + + #[Test] + public function it_generates_a_large_report_in_chunks() + { + config(['statamic.seo-pro.reports.queue_chunk_size' => 3]); + + $this->generateEntries(7); + + Report::create()->save()->generate(); + + $this->assertDatabaseCount('seo_pro_report_pages', 7); + $this->assertDatabaseCount('seo_pro_report_chunks', 0); + $this->assertTrue(Report::find(1)->isGenerated()); + $this->assertSame(7, Report::find(1)->pagesCrawled()); + } +} diff --git a/tests/Reporting/Eloquent/PublishReportMigrationsTest.php b/tests/Reporting/Eloquent/PublishReportMigrationsTest.php new file mode 100644 index 00000000..da224d09 --- /dev/null +++ b/tests/Reporting/Eloquent/PublishReportMigrationsTest.php @@ -0,0 +1,97 @@ +each->delete(); + Term::all()->each->delete(); + + if ($this->files->exists($path = storage_path('statamic/seopro/reports'))) { + $this->files->deleteDirectory($path); + } + + $this->deletePublishedMigrations(); + } + + protected function tearDown(): void + { + $this->deletePublishedMigrations(); + + parent::tearDown(); + } + + #[Test] + public function it_publishes_the_migration_and_imports_file_reports() + { + $this->generateEntries(5); + + Report::create()->save()->generate(); + $this->assertFileExists(storage_path('statamic/seopro/reports/1/report.yaml')); + + $this + ->artisan('statamic:seo-pro:database-reports') + ->expectsConfirmation('Would you like to import existing reports?', 'yes') + ->assertSuccessful(); + + $this->assertCount(1, $this->publishedMigrations()); + $this->assertDatabaseHas('seo_pro_reports', ['id' => 1, 'status' => 'fail', 'pages_crawled' => 5]); + $this->assertDatabaseCount('seo_pro_report_pages', 5); + } + + #[Test] + public function it_imports_nothing_when_there_are_no_file_reports() + { + $this + ->artisan('statamic:seo-pro:database-reports') + ->expectsConfirmation('Would you like to import existing reports?', 'yes') + ->expectsOutputToContain('no reports to import') + ->assertSuccessful(); + + $this->assertDatabaseCount('seo_pro_reports', 0); + } + + #[Test] + public function it_does_not_publish_the_migration_twice() + { + $this + ->artisan('statamic:seo-pro:database-reports') + ->expectsConfirmation('Would you like to import existing reports?', 'no') + ->assertSuccessful(); + + $this + ->artisan('statamic:seo-pro:database-reports') + ->expectsOutputToContain('already exists') + ->expectsConfirmation('Would you like to import existing reports?', 'no') + ->assertSuccessful(); + + $this->assertCount(1, $this->publishedMigrations()); + } + + private function publishedMigrations() + { + return collect($this->files->allFiles(database_path('migrations'))) + ->filter(fn ($file) => str_contains($file->getFilename(), 'create_seo_pro_reports_tables')) + ->values(); + } + + private function deletePublishedMigrations() + { + $this->publishedMigrations()->each(fn ($file) => $this->files->delete($file->getPathname())); + } +} diff --git a/tests/Reporting/Eloquent/ReportControllerTest.php b/tests/Reporting/Eloquent/ReportControllerTest.php new file mode 100644 index 00000000..df88a94c --- /dev/null +++ b/tests/Reporting/Eloquent/ReportControllerTest.php @@ -0,0 +1,44 @@ +set('statamic.seo-pro.reports.driver', 'database'); + } + + protected function defineDatabaseMigrations() + { + $this->loadMigrationsFrom(__DIR__.'/../../../src/Commands/stubs'); + } + + #[Test] + public function create_redirects_to_the_new_report_and_show_finds_it() + { + $this->actingAs(User::make()->makeSuper()->save()); + + $this->get(cp_route('seo-pro.reports.create'))->assertRedirect(cp_route('seo-pro.reports.show', 1)); + $this->assertDatabaseHas('seo_pro_reports', ['id' => 1]); + + $this->get(cp_route('seo-pro.reports.show', 1))->assertOk(); + $this->get(cp_route('seo-pro.reports.show', 99))->assertNotFound(); + $this->getJson(cp_route('seo-pro.reports.pages.index', 1))->assertOk(); + $this->get(cp_route('seo-pro.reports.index'))->assertOk(); + + $this->delete(cp_route('seo-pro.reports.destroy', 1)); + $this->assertDatabaseMissing('seo_pro_reports', ['id' => 1]); + } +} diff --git a/tests/Reporting/Eloquent/ReportRepositoryTest.php b/tests/Reporting/Eloquent/ReportRepositoryTest.php new file mode 100644 index 00000000..ba97b236 --- /dev/null +++ b/tests/Reporting/Eloquent/ReportRepositoryTest.php @@ -0,0 +1,253 @@ +repo = $this->app->make(ReportRepository::class); + } + + protected function getEnvironmentSetUp($app) + { + parent::getEnvironmentSetUp($app); + + $app['config']->set('statamic.seo-pro.reports.driver', 'database'); + } + + protected function defineDatabaseMigrations() + { + $this->loadMigrationsFrom(__DIR__.'/../../../src/Commands/stubs'); + } + + #[Test] + public function it_binds_the_eloquent_repository_for_the_database_driver() + { + $this->assertInstanceOf(ReportRepository::class, $this->app->make(ReportRepositoryContract::class)); + } + + #[Test] + public function can_find_report() + { + $this->createReportModel(); + + $report = $this->repo->find(1); + + $this->assertInstanceOf(Report::class, $report); + $this->assertSame(1, $report->id()); + $this->assertSame(1700000000, $report->date()->timestamp); + $this->assertSame('pending', $report->status()); + } + + #[Test] + public function it_returns_null_for_unknown_ids() + { + $this->assertNull($this->repo->find(99)); + $this->assertNull($this->repo->find('abc')); + } + + #[Test] + public function can_save_report() + { + $this->repo->save($this->pendingReport()); + + $this->assertDatabaseHas('seo_pro_reports', ['id' => 1, 'status' => 'pending']); + } + + #[Test] + public function can_delete_report() + { + $this->createReportModel(); + + $this->repo->delete($this->repo->find(1)); + + $this->assertDatabaseMissing('seo_pro_reports', ['id' => 1]); + } + + #[Test] + public function it_lists_reports_newest_first() + { + $this->createReportModel(); + $this->createReportModel(); + + $this->assertSame([2, 1], $this->repo->all()->map->id()->all()); + } + + #[Test] + public function it_finds_the_latest_report() + { + $this->assertNull($this->repo->latest()); + + $this->createReportModel(); + $this->createReportModel(); + + $this->assertSame(2, $this->repo->latest()->id()); + } + + #[Test] + public function it_computes_the_next_id() + { + $this->assertSame(1, $this->repo->nextId()); + + $this->createReportModel(); + + $this->assertSame(2, $this->repo->nextId()); + } + + #[Test] + public function can_save_chunk() + { + $report = $this->savedReport(); + + $this->repo->saveChunk(new Chunk(1, ['a', 'b'], $report)); + + $this->assertDatabaseHas('seo_pro_report_chunks', ['report_id' => 1, 'chunk_id' => 1]); + $this->assertTrue($this->repo->hasChunks($report)); + } + + #[Test] + public function can_delete_chunk() + { + $report = $this->savedReport(); + $this->repo->saveChunk(new Chunk(1, ['a', 'b'], $report)); + + $this->repo->deleteChunk(new Chunk(1, ['a', 'b'], $report)); + + $this->assertFalse($this->repo->hasChunks($report)); + } + + #[Test] + public function can_save_page() + { + $report = $this->savedReport(); + $page = new Page('entry-1', ['canonical_url' => 'http://x/a', 'site' => 'default'], $report); + + $this->repo->savePage($page); + $this->repo->savePage($page->setResults(['SiteName' => true])); + + $this->assertDatabaseCount('seo_pro_report_pages', 1); + $this->assertDatabaseHas('seo_pro_report_pages', ['report_id' => 1, 'page_id' => 'entry-1']); + } + + #[Test] + public function it_saves_a_page_in_one_query() + { + $report = $this->savedReport(); + $page = (new Page('entry-1', ['canonical_url' => 'http://x/a'], $report))->setResults(['SiteName' => true]); + + DB::enableQueryLog(); + $this->repo->savePage($page); + $this->repo->savePage($page); + + $this->assertCount(2, DB::getQueryLog()); + $this->assertDatabaseCount('seo_pro_report_pages', 1); + } + + #[Test] + public function it_loads_pages_for_a_report() + { + $report = $this->savedReport(); + ReportPageModel::create([ + 'report_id' => 1, + 'page_id' => 'entry-1', + 'data' => ['canonical_url' => 'http://x/a', 'site' => 'default'], + 'results' => ['SiteName' => true], + ]); + + $pages = $this->repo->pages($report); + + $this->assertCount(1, $pages); + $this->assertSame('entry-1', $pages->first()->id()); + $this->assertSame('http://x/a', $pages->first()->url()); + $this->assertSame(['SiteName' => true], $pages->first()->results()); + $this->assertSame($report, $pages->first()->report()); + } + + #[Test] + public function it_deletes_pages_and_chunks_with_the_report() + { + $report = $this->savedReport(); + ReportPageModel::create(['report_id' => 1, 'page_id' => 'entry-1', 'data' => [], 'results' => []]); + ReportChunkModel::create(['report_id' => 1, 'chunk_id' => 1, 'content_ids' => ['a']]); + + $this->repo->delete($report); + + $this->assertDatabaseCount('seo_pro_report_pages', 0); + $this->assertDatabaseCount('seo_pro_report_chunks', 0); + } + + #[Test] + public function it_keeps_the_date_outside_utc() + { + $timezone = date_default_timezone_get(); + date_default_timezone_set('Europe/Berlin'); + + try { + $this->repo->save($this->pendingReport()); + + $this->assertSame(1700000000, $this->repo->find(1)->date()->timestamp); + } finally { + date_default_timezone_set($timezone); + } + } + + #[Test] + public function it_keeps_the_newest_reports_when_deleting_old_ones() + { + config(['statamic.seo-pro.reports.keep_recent' => 1]); + + Report::create()->save(); + Report::create()->save(); + + $this->assertSame([2], Report::all()->map->id()->all()); + } + + private function createReportModel() + { + return ReportModel::create([ + 'date' => '2023-11-14 22:13:20', + 'status' => 'pending', + ]); + } + + private function pendingReport() + { + return Report::create()->hydrate([ + 'date' => 1700000000, + 'status' => 'pending', + 'score' => null, + 'pages_crawled' => null, + 'pages_actionable' => null, + 'results' => null, + ]); + } + + private function savedReport() + { + $report = $this->pendingReport(); + + $this->repo->save($report); + + return $report; + } +} diff --git a/tests/Reporting/File/ReportRepositoryTest.php b/tests/Reporting/File/ReportRepositoryTest.php new file mode 100644 index 00000000..36ce2d38 --- /dev/null +++ b/tests/Reporting/File/ReportRepositoryTest.php @@ -0,0 +1,210 @@ +files->exists($path = $this->reportsPath())) { + $this->files->deleteDirectory($path); + } + + $this->repo = $this->app->make(ReportRepository::class); + } + + protected function tearDown(): void + { + if ($this->files->exists($path = $this->elsewherePath())) { + $this->files->deleteDirectory($path); + } + + parent::tearDown(); + } + + #[Test] + public function it_binds_the_file_repository_by_default() + { + $this->assertInstanceOf(ReportRepository::class, $this->app->make(ReportRepositoryContract::class)); + } + + #[Test] + public function can_find_report() + { + $this->repo->save($this->pendingReport()); + + $report = $this->repo->find(1); + + $this->assertInstanceOf(Report::class, $report); + $this->assertSame(1, $report->id()); + $this->assertSame(1700000000, $report->date()->timestamp); + $this->assertSame('pending', $report->status()); + } + + #[Test] + public function it_returns_null_for_unknown_ids() + { + $this->assertNull($this->repo->find(99)); + $this->assertNull($this->repo->find('abc')); + } + + #[Test] + public function can_save_report() + { + $this->repo->save($this->pendingReport()); + + $this->assertFileExists($this->reportsPath('1/report.yaml')); + } + + #[Test] + public function can_delete_report() + { + $report = $this->savedReport(); + + $this->repo->delete($report); + + $this->assertFileDoesNotExist($this->reportsPath('1')); + } + + #[Test] + public function it_lists_reports_newest_first() + { + $this->repo->save($this->pendingReport()); + $this->repo->save($this->pendingReport()); + + $this->assertSame([2, 1], $this->repo->all()->map->id()->all()); + } + + #[Test] + public function it_finds_the_latest_report() + { + $this->assertNull($this->repo->latest()); + + $this->repo->save($this->pendingReport()); + $this->repo->save($this->pendingReport()); + + $this->assertSame(2, $this->repo->latest()->id()); + } + + #[Test] + public function it_computes_the_next_id() + { + $this->assertSame(1, $this->repo->nextId()); + + $this->repo->save($this->pendingReport()); + + $this->assertSame(2, $this->repo->nextId()); + } + + #[Test] + public function it_reads_the_directory_from_config() + { + config(['statamic.seo-pro.reports.directory' => $this->elsewherePath()]); + + $this->repo->save($this->pendingReport()); + + $this->assertFileExists($this->elsewherePath('1/report.yaml')); + } + + #[Test] + public function it_falls_back_to_the_default_directory_when_the_key_is_missing() + { + // A config published before the directory key existed has a reports section without it. + config(['statamic.seo-pro.reports' => ['keep_recent' => 'all', 'queue_chunk_size' => 1000]]); + + $this->repo->save($this->pendingReport()); + + $this->assertFileExists($this->reportsPath('1/report.yaml')); + $this->assertCount(1, $this->repo->all()); + } + + #[Test] + public function can_save_chunk() + { + $report = $this->savedReport(); + + $this->repo->saveChunk(new Chunk(1, ['a', 'b'], $report)); + + $this->assertFileExists($this->reportsPath('1/chunks/1/chunk.yaml')); + $this->assertTrue($this->repo->hasChunks($report)); + } + + #[Test] + public function can_delete_chunk() + { + $report = $this->savedReport(); + $this->repo->saveChunk(new Chunk(1, ['a', 'b'], $report)); + + $this->repo->deleteChunk(new Chunk(1, ['a', 'b'], $report)); + + $this->assertFalse($this->repo->hasChunks($report)); + } + + #[Test] + public function can_save_page() + { + $report = $this->savedReport(); + + $this->repo->savePage((new Page('entry-1', ['canonical_url' => 'http://x/a', 'site' => 'default'], $report))->setResults(['SiteName' => true])); + + $this->assertCount(1, $this->files->allFiles($this->reportsPath('1/pages'))); + } + + #[Test] + public function it_loads_pages_for_a_report() + { + $report = $this->savedReport(); + $this->repo->savePage((new Page('entry-1', ['canonical_url' => 'http://x/a', 'site' => 'default'], $report))->setResults(['SiteName' => true])); + + $pages = $this->repo->pages($report); + + $this->assertCount(1, $pages); + $this->assertSame('entry-1', $pages->first()->id()); + $this->assertSame('http://x/a', $pages->first()->url()); + $this->assertSame(['SiteName' => true], $pages->first()->results()); + $this->assertSame($report, $pages->first()->report()); + } + + private function reportsPath($append = null) + { + return collect([storage_path('statamic/seopro/reports'), $append])->filter()->implode('/'); + } + + private function elsewherePath($append = null) + { + return collect([storage_path('statamic/seopro/elsewhere'), $append])->filter()->implode('/'); + } + + private function pendingReport() + { + return Report::create()->hydrate([ + 'date' => 1700000000, + 'status' => 'pending', + 'score' => null, + 'pages_crawled' => null, + 'pages_actionable' => null, + 'results' => null, + ]); + } + + private function savedReport() + { + $report = $this->pendingReport(); + + $this->repo->save($report); + + return $report; + } +} diff --git a/tests/Reporting/GeneratesContent.php b/tests/Reporting/GeneratesContent.php new file mode 100644 index 00000000..064ff910 --- /dev/null +++ b/tests/Reporting/GeneratesContent.php @@ -0,0 +1,36 @@ +each(function ($i) { + Entry::make() + ->collection('articles') + ->blueprint('article') + ->slug('test-entry-'.$i) + ->set('title', 'Test Entry '.$i) + ->save(); + }); + + return $this; + } + + protected function generateTerms($count) + { + collect(range(1, $count))->each(function ($i) { + Term::make() + ->slug('test-term-'.$i) + ->taxonomy('topics') + ->set('title', 'Test Term '.$i) + ->save(); + }); + + return $this; + } +}