From c7b4ec25af9d2b0d183759d8bc0e3892c2a7450a Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 3 Aug 2026 15:54:55 +0200 Subject: [PATCH] feat: add manage:list-external-buckets command Read-only stack-wide audit of external buckets (buckets with hasExternalSchema), reporting created date, the isReadOnly flag, whether the bucket is linked from elsewhere, and whether KBC.description metadata is set. Written while investigating why descriptions were missing on older external buckets (DMD-1496), where the answer turned out to be isReadOnly on buckets created before the flag changed meaning. Output is CSV on stdout so it can be redirected straight to a file; progress and summary lines are prefixed with #. Creates a 15-minute Storage token per project and drops it in a finally block, so a listing failure does not leak tokens. Projects the manage token cannot reach are reported and counted as skipped rather than aborting the run, which matters on stacks where not every project is accessible. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 22 +++ cli.php | 2 + .../Console/Command/ListExternalBuckets.php | 143 ++++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 src/Keboola/Console/Command/ListExternalBuckets.php diff --git a/README.md b/README.md index 006ae71..53b23e5 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,28 @@ cat data.csv | php cli.php storage:notify-projects MANAGETOKEN ``` +### List external buckets on a stack +Read-only. Walks every project on the stack and prints a CSV of all external buckets (buckets with +`hasExternalSchema`), including whether each one is read-only, whether it is linked from elsewhere, +and whether it has `KBC.description` metadata set. + +``` +php cli.php manage:list-external-buckets [] +``` +Arguments: +- manage-token (required) Manage API token (super admin); used to create short-lived project Storage tokens. +- hostname-suffix (optional, default: keboola.com) Connection host suffix (e.g. eu-central-1.keboola.com). + +Behavior: +- Iterates maintainers -> organizations -> projects, de-duplicating projects that appear more than once. +- Creates a 15-minute Storage token per project and always drops it afterwards, including when + listing fails. +- Projects the token cannot reach are reported as comment lines and counted as skipped, so the run + continues over the rest of the stack. +- Writes CSV to stdout with the header + `projectId,projectName,bucketId,created,isReadOnly,linked,hasDescription,description`, so it can be + redirected straight to a file. Progress and summary lines are prefixed with `#`. + ### Force Unlink Shared and Linked Buckets List all buckets in the project and force-unlink those that are both shared and linked. By default, the command runs in dry-run mode and only reports what would be unlinked. Use the `--force` flag to actually perform the unlinking. diff --git a/cli.php b/cli.php index 2db2c82..89a1684 100644 --- a/cli.php +++ b/cli.php @@ -15,6 +15,7 @@ use Keboola\Console\Command\DeleteOwnerlessWorkspaces; use Keboola\Console\Command\DescribeOrganizationWorkspaces; use Keboola\Console\Command\LineageEventsExport; +use Keboola\Console\Command\ListExternalBuckets; use Keboola\Console\Command\MassDeleteProjectWorkspaces; use Keboola\Console\Command\MassProjectEnableDynamicBackends; use Keboola\Console\Command\MassProjectExtendExpiration; @@ -64,6 +65,7 @@ $application->add(new UpdateDataRetention()); $application->add(new OrganizationResetWorkspacePasswords()); $application->add(new ForceUnlinkSharedBuckets()); +$application->add(new ListExternalBuckets()); $application->add(new OrganizationsAddFeature()); $application->add(new DeleteProjects()); $application->add(new DeleteStorageBackend()); diff --git a/src/Keboola/Console/Command/ListExternalBuckets.php b/src/Keboola/Console/Command/ListExternalBuckets.php new file mode 100644 index 0000000..9f92d24 --- /dev/null +++ b/src/Keboola/Console/Command/ListExternalBuckets.php @@ -0,0 +1,143 @@ +setName('manage:list-external-buckets') + ->setDescription( + 'Read-only: list all external buckets (hasExternalSchema) across all projects of a stack ' + . 'with created date, isReadOnly flag and whether KBC.description metadata is set.' + ) + ->addArgument( + self::ARGUMENT_MANAGE_TOKEN, + InputArgument::REQUIRED, + 'Manage API token (super admin) used to create short-lived project storage tokens.' + ) + ->addArgument( + self::ARGUMENT_HOSTNAME_SUFFIX, + InputArgument::OPTIONAL, + 'Keboola Connection Hostname Suffix', + 'keboola.com' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $manageToken = $input->getArgument(self::ARGUMENT_MANAGE_TOKEN); + assert(is_string($manageToken)); + $hostnameSuffix = $input->getArgument(self::ARGUMENT_HOSTNAME_SUFFIX); + assert(is_string($hostnameSuffix)); + assert($hostnameSuffix !== ''); + + $serviceClient = new ServiceClient($hostnameSuffix); + $connectionUrl = $serviceClient->getConnectionServiceUrl(); + $manageClient = new ManageClient(['token' => $manageToken, 'url' => $connectionUrl]); + + $output->writeln('projectId,projectName,bucketId,created,isReadOnly,linked,hasDescription,description'); + + $externalBucketsFound = 0; + $projectsChecked = 0; + $projectsSkipped = 0; + $seenProjects = []; + + foreach ($manageClient->listMaintainers() as $maintainer) { + foreach ($manageClient->listMaintainerOrganizations($maintainer['id']) as $organization) { + foreach ($manageClient->listOrganizationProjects($organization['id']) as $project) { + $projectId = (int) $project['id']; + if (isset($seenProjects[$projectId])) { + continue; + } + $seenProjects[$projectId] = true; + + try { + $storageToken = $manageClient->createProjectStorageToken( + $projectId, + [ + 'description' => 'List external buckets (read-only audit)', + 'expiresIn' => 900, + 'canManageBuckets' => true, + ] + ); + } catch (\Throwable $e) { + $output->writeln(sprintf( + '# Access denied or error for project "%s" ("%s"): %s', + $projectId, + $project['name'], + $e->getMessage(), + )); + $projectsSkipped++; + continue; + } + assert(is_string($storageToken['token'])); + $projectsChecked++; + + $storageClient = new StorageApiClient([ + 'token' => $storageToken['token'], + 'url' => $connectionUrl, + ]); + + try { + $buckets = $storageClient->listBuckets(['include' => 'metadata']); + foreach ($buckets as $bucket) { + if (($bucket['hasExternalSchema'] ?? false) !== true) { + continue; + } + $externalBucketsFound++; + + $description = null; + foreach ($bucket['metadata'] ?? [] as $metadata) { + if ($metadata['key'] === 'KBC.description') { + $description = (string) $metadata['value']; + break; + } + } + + $output->writeln(sprintf( + '%d,"%s",%s,%s,%s,%s,%s,"%s"', + $projectId, + str_replace('"', '""', (string) $project['name']), + $bucket['id'], + $bucket['created'], + ($bucket['isReadOnly'] ?? false) ? 'READ_ONLY' : 'writable', + isset($bucket['sourceBucket']) ? 'linked' : 'own', + $description !== null ? 'yes' : 'no', + str_replace('"', '""', (string) ($description ?? '')), + )); + } + } finally { + $tokensClient = new Tokens($storageClient); + assert(is_scalar($storageToken['id'])); + $tokensClient->dropToken((int) $storageToken['id']); + } + } + } + } + + $output->writeln(sprintf( + '# Done: %d external buckets in %d projects checked (%d projects skipped).', + $externalBucketsFound, + $projectsChecked, + $projectsSkipped, + )); + + return 0; + } +}