From bb2355b0de04d96779805883ac74f49896c4578c Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Fri, 7 Aug 2026 16:25:13 +0200 Subject: [PATCH] feat: add manage:list-backends command with cleanup verdicts Read-only inventory of a stack's storage backends. Fetches the detail of every backend to include loginType, keyPairLastRotatedAt, SSO and dynamic backend flags, assigns a cleanup verdict per backend and prints ID lists ready for manage:delete-backend. Used for the DMD-1638 prod cleanup. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 34 ++ cli.php | 2 + .../Console/Command/ListStorageBackends.php | 348 ++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 src/Keboola/Console/Command/ListStorageBackends.php diff --git a/README.md b/README.md index f95a596..1e52df3 100644 --- a/README.md +++ b/README.md @@ -669,6 +669,40 @@ You can use it to set all projects of an organization to use a storage backend. php ./cli.php manage:set-organization-storage-backend [--force/-f] ``` +## List Storage Backends +Read-only inventory of a stack's storage backends with a cleanup verdict per backend. Nothing is modified. +Pairs with `manage:delete-backend` — the ID lists printed at the end are ready to paste into it. + +- Run the command + ``` + php ./cli.php manage:list-backends [--format=table|csv] [--verdict=VERDICT] [--unsupported] + ``` +Arguments: +- manage-token (required) Manage API token for the stack. +- stack-url (required) Full stack URL, e.g. https://connection.keboola.com. + +Options: +- --format Output format, `table` (default) or `csv`. In `csv` mode the summary goes to stderr so stdout stays parseable. +- --verdict Show only backends with the given verdict. +- --unsupported Show only backends that are not snowflake/bigquery — i.e. both `DELETE_UNSUPPORTED_UNUSED` and `REVIEW_UNSUPPORTED_IN_USE` in one pass. Composable with `--verdict`. + +Verdicts (backends we keep are snowflake and bigquery; everything else is a decommissioned type +such as mysql/redshift/synapse/exasol/teradata, or a parked PoC such as supabase): +- `DELETE_UNSUPPORTED_UNUSED` — not snowflake/bigquery, 0 projects and 0 maintainers. Safe to delete. +- `DELETE_UNUSED` — snowflake/bigquery, 0 projects and 0 maintainers. +- `REVIEW_UNSUPPORTED_IN_USE` — not snowflake/bigquery, but still has projects or maintainers. +- `REVIEW_MAINTAINER_ONLY` — 0 projects, but a maintainer still points at it as its default backend. +- `KEEP` — has live projects. + +Counts come from `assignedProjectsCount` / `assignedMaintainersCount` and cover live projects only. +A backend showing 0 can still be blocked by soft-deleted, not-yet-purged projects — the delete guard refuses those. + +The command fetches the detail of every backend (`GET /manage/storage-backend/{id}`) to add columns that are +not in the list response: `loginType`, `keyRotated` (= `keyPairLastRotatedAt`, date only), `dynBackends` +(= `useDynamicBackends`), `useSso`, `ssoEnabled`, `ssoConfigured`. Booleans render as 1/0, empty when the +detail does not report the field (e.g. BigQuery has no `loginType`). A failed detail call leaves the extra +columns empty and keeps the row. + ## Delete Storage Backend Delete one or more storage backends from a stack by their IDs. Dry-run by default. diff --git a/cli.php b/cli.php index 2908e23..8a26a0a 100644 --- a/cli.php +++ b/cli.php @@ -17,6 +17,7 @@ use Keboola\Console\Command\DeleteOwnerlessWorkspaces; use Keboola\Console\Command\DescribeOrganizationWorkspaces; use Keboola\Console\Command\LineageEventsExport; +use Keboola\Console\Command\ListStorageBackends; use Keboola\Console\Command\MassDeleteProjectWorkspaces; use Keboola\Console\Command\MassProjectEnableDynamicBackends; use Keboola\Console\Command\MassProjectExtendExpiration; @@ -71,4 +72,5 @@ $application->add(new OrganizationsAddFeature()); $application->add(new DeleteProjects()); $application->add(new DeleteStorageBackend()); +$application->add(new ListStorageBackends()); $application->run(); diff --git a/src/Keboola/Console/Command/ListStorageBackends.php b/src/Keboola/Console/Command/ListStorageBackends.php new file mode 100644 index 0000000..7dc29ea --- /dev/null +++ b/src/Keboola/Console/Command/ListStorageBackends.php @@ -0,0 +1,348 @@ +setName('manage:list-backends') + ->setDescription( + 'Read-only: list all storage backends of a stack with project/maintainer counts ' + . 'and a cleanup verdict. Prints ID lists ready for manage:delete-backend.' + ) + ->addArgument(self::ARGUMENT_TOKEN, InputArgument::REQUIRED, 'manage api token') + ->addArgument(self::ARGUMENT_URL, InputArgument::REQUIRED, 'Stack URL') + ->addOption( + self::OPTION_FORMAT, + null, + InputOption::VALUE_REQUIRED, + 'Output format: table or csv', + self::FORMAT_TABLE + ) + ->addOption( + self::OPTION_VERDICT, + null, + InputOption::VALUE_REQUIRED, + 'Show only backends with this verdict (e.g. DELETE_UNSUPPORTED_UNUSED)' + ) + ->addOption( + self::OPTION_UNSUPPORTED, + null, + InputOption::VALUE_NONE, + 'Show only backends that are not snowflake/bigquery' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $token = $input->getArgument(self::ARGUMENT_TOKEN); + assert(is_string($token)); + $url = $input->getArgument(self::ARGUMENT_URL); + assert(is_string($url)); + + $format = $input->getOption(self::OPTION_FORMAT); + if (!in_array($format, [self::FORMAT_TABLE, self::FORMAT_CSV], true)) { + $output->writeln('Unknown format, use table or csv'); + return 1; + } + $verdictFilter = $input->getOption(self::OPTION_VERDICT); + assert($verdictFilter === null || is_string($verdictFilter)); + $unsupportedOnly = (bool) $input->getOption(self::OPTION_UNSUPPORTED); + + $client = new Client(['url' => $url, 'token' => $token]); + $guzzle = new GuzzleClient([ + 'base_uri' => $url, + 'headers' => ['X-KBC-ManageApiToken' => $token], + ]); + + $backends = $client->listStorageBackend(); + assert(is_array($backends)); + + $progress = $output instanceof ConsoleOutputInterface ? $output->getErrorOutput() : $output; + $progress->writeln(sprintf('Fetching details for %d backends...', count($backends))); + + $rows = []; + foreach ($backends as $backend) { + assert(is_array($backend)); + $rows[] = $this->buildRow(array_merge($backend, $this->fetchDetail($guzzle, $backend))); + } + usort($rows, fn (array $a, array $b) => $a['id'] <=> $b['id']); + + $visibleRows = $this->filterRows($rows, $unsupportedOnly, $verdictFilter); + + if ($format === self::FORMAT_CSV) { + $this->renderCsv($output, $visibleRows); + } else { + $this->renderTable($output, $visibleRows); + } + + $this->renderSummary($output, $rows, $format === self::FORMAT_CSV); + + return 0; + } + + /** + * @param array $backend + * @return array + */ + private function fetchDetail(GuzzleClient $guzzle, array $backend): array + { + try { + $response = $guzzle->get(sprintf('manage/storage-backend/%d', $this->asInt($backend['id'] ?? 0))); + $detail = json_decode((string) $response->getBody(), true); + return is_array($detail) ? $detail : []; + } catch (Throwable) { + // Keep the list row usable even when a single detail call fails. + return []; + } + } + + /** + * @param array $backend + * @return array{id: int, backend: string, host: string, region: string, owner: string, + * technicalOwner: string, projects: int, maintainers: int, buckets: int, loginType: string, + * keyRotated: string, dynBackends: string, useSso: string, ssoEnabled: string, + * ssoConfigured: string, created: string, verdict: string} + */ + private function buildRow(array $backend): array + { + $stats = $backend['stats'] ?? []; + assert(is_array($stats)); + + $type = $this->asString($backend['backend'] ?? ''); + $projects = $this->asInt($backend['assignedProjectsCount'] ?? 0); + $maintainers = $this->asInt($backend['assignedMaintainersCount'] ?? 0); + + return [ + 'id' => $this->asInt($backend['id'] ?? 0), + 'backend' => $type, + // BigQuery backends have no host, they report a folderId instead. + 'host' => $this->asString($backend['host'] ?? $backend['folderId'] ?? ''), + 'region' => $this->asString($backend['region'] ?? ''), + 'owner' => $this->asString($backend['owner'] ?? ''), + 'technicalOwner' => $this->asString($backend['technicalOwner'] ?? ''), + 'projects' => $projects, + 'maintainers' => $maintainers, + 'buckets' => $this->asInt($stats['bucketsCount'] ?? 0), + 'loginType' => $this->asString($backend['loginType'] ?? ''), + 'keyRotated' => substr($this->asString($backend['keyPairLastRotatedAt'] ?? ''), 0, 10), + 'dynBackends' => $this->asBool($backend['useDynamicBackends'] ?? null), + 'useSso' => $this->asBool($backend['useSso'] ?? null), + 'ssoEnabled' => $this->asBool($backend['isSsoEnabled'] ?? null), + 'ssoConfigured' => $this->asBool($backend['isSsoConfigured'] ?? null), + 'created' => substr($this->asString($backend['created'] ?? ''), 0, 10), + 'verdict' => $this->resolveVerdict($type, $projects, $maintainers), + ]; + } + + /** + * @param array> $rows + * @return array> + */ + private function filterRows(array $rows, bool $unsupportedOnly, ?string $verdict): array + { + if ($unsupportedOnly) { + $rows = array_filter($rows, fn (array $r) => !in_array($r['backend'], self::KEPT_BACKENDS, true)); + } + if ($verdict !== null) { + $rows = array_filter($rows, fn (array $r) => $r['verdict'] === $verdict); + } + return array_values($rows); + } + + private function resolveVerdict(string $type, int $projects, int $maintainers): string + { + $isUnsupported = !in_array($type, self::KEPT_BACKENDS, true); + $isUnused = $projects === 0 && $maintainers === 0; + + if ($isUnsupported) { + return $isUnused ? self::VERDICT_DELETE_UNSUPPORTED_UNUSED : self::VERDICT_REVIEW_UNSUPPORTED_IN_USE; + } + if ($isUnused) { + return self::VERDICT_DELETE_UNUSED; + } + if ($projects === 0) { + // No projects, but a maintainer still points at it as its default backend. + return self::VERDICT_REVIEW_MAINTAINER_ONLY; + } + return self::VERDICT_KEEP; + } + + private function asString(mixed $value): string + { + return is_scalar($value) ? (string) $value : ''; + } + + private function asInt(mixed $value): int + { + return is_numeric($value) ? (int) $value : 0; + } + + private function asBool(mixed $value): string + { + if ($value === null) { + return ''; + } + return $value ? '1' : '0'; + } + + /** + * @param array> $rows + */ + private function renderCsv(OutputInterface $output, array $rows): void + { + $output->writeln(implode(',', self::COLUMNS)); + foreach ($rows as $row) { + $cells = []; + foreach (self::COLUMNS as $column) { + $cells[] = '"' . str_replace('"', '""', (string) $row[$column]) . '"'; + } + $output->writeln(implode(',', $cells)); + } + } + + /** + * @param array> $rows + */ + private function renderTable(OutputInterface $output, array $rows): void + { + $table = new Table($output); + $table->setHeaders(self::COLUMNS); + foreach ($rows as $row) { + $cells = []; + foreach (self::COLUMNS as $column) { + $cells[] = (string) $row[$column]; + } + $table->addRow($cells); + } + $table->render(); + } + + /** + * @param array> $rows + */ + private function renderSummary(OutputInterface $output, array $rows, bool $toStdErr): void + { + // In CSV mode keep stdout parseable — the summary goes to stderr. + $out = $toStdErr && $output instanceof ConsoleOutputInterface + ? $output->getErrorOutput() + : $output; + + $byType = []; + $byVerdict = []; + foreach ($rows as $row) { + $byType[(string) $row['backend']] = ($byType[(string) $row['backend']] ?? 0) + 1; + $byVerdict[(string) $row['verdict']] = ($byVerdict[(string) $row['verdict']] ?? 0) + 1; + } + arsort($byType); + arsort($byVerdict); + + $out->writeln(''); + $out->writeln(sprintf('Total backends: %d', count($rows))); + $out->writeln(''); + $out->writeln('By backend type:'); + foreach ($byType as $type => $count) { + $out->writeln(sprintf(' %-12s %d', $type, $count)); + } + $out->writeln(''); + $out->writeln('By verdict:'); + foreach ($byVerdict as $verdict => $count) { + $out->writeln(sprintf(' %-24s %d', $verdict, $count)); + } + + $out->writeln(''); + $out->writeln('Delete candidates (paste into manage:delete-backend):'); + foreach ([self::VERDICT_DELETE_UNSUPPORTED_UNUSED, self::VERDICT_DELETE_UNUSED] as $verdict) { + $ids = []; + foreach ($rows as $row) { + if ($row['verdict'] === $verdict) { + $ids[] = (string) $row['id']; + } + } + $out->writeln(''); + $out->writeln(sprintf(' %s (%d):', $verdict, count($ids))); + $out->writeln($ids === [] ? ' -' : ' ' . implode(',', $ids)); + } + + $reviewVerdicts = [self::VERDICT_REVIEW_UNSUPPORTED_IN_USE, self::VERDICT_REVIEW_MAINTAINER_ONLY]; + $needsReview = 0; + foreach ($rows as $row) { + if (in_array($row['verdict'], $reviewVerdicts, true)) { + $needsReview++; + } + } + if ($needsReview > 0) { + $out->writeln(''); + $out->writeln(sprintf( + '%d backend(s) need a manual look before deletion (REVIEW_* verdicts above).', + $needsReview + )); + } + $out->writeln(''); + $out->writeln( + 'Note: counts cover live projects only. A backend showing 0 can still be blocked by ' + . 'soft-deleted, not-yet-purged projects — the delete guard will refuse those.' + ); + } +}