diff --git a/README.md b/README.md index 006ae71..53b23e5 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,28 @@ cat data.csv | php cli.php storage:notify-projects MANAGETOKEN ``` +### List external buckets on a stack +Read-only. Walks every project on the stack and prints a CSV of all external buckets (buckets with +`hasExternalSchema`), including whether each one is read-only, whether it is linked from elsewhere, +and whether it has `KBC.description` metadata set. + +``` +php cli.php manage:list-external-buckets [] +``` +Arguments: +- manage-token (required) Manage API token (super admin); used to create short-lived project Storage tokens. +- hostname-suffix (optional, default: keboola.com) Connection host suffix (e.g. eu-central-1.keboola.com). + +Behavior: +- Iterates maintainers -> organizations -> projects, de-duplicating projects that appear more than once. +- Creates a 15-minute Storage token per project and always drops it afterwards, including when + listing fails. +- Projects the token cannot reach are reported as comment lines and counted as skipped, so the run + continues over the rest of the stack. +- Writes CSV to stdout with the header + `projectId,projectName,bucketId,created,isReadOnly,linked,hasDescription,description`, so it can be + redirected straight to a file. Progress and summary lines are prefixed with `#`. + ### Force Unlink Shared and Linked Buckets List all buckets in the project and force-unlink those that are both shared and linked. By default, the command runs in dry-run mode and only reports what would be unlinked. Use the `--force` flag to actually perform the unlinking. diff --git a/cli.php b/cli.php index 2db2c82..89a1684 100644 --- a/cli.php +++ b/cli.php @@ -15,6 +15,7 @@ use Keboola\Console\Command\DeleteOwnerlessWorkspaces; use Keboola\Console\Command\DescribeOrganizationWorkspaces; use Keboola\Console\Command\LineageEventsExport; +use Keboola\Console\Command\ListExternalBuckets; use Keboola\Console\Command\MassDeleteProjectWorkspaces; use Keboola\Console\Command\MassProjectEnableDynamicBackends; use Keboola\Console\Command\MassProjectExtendExpiration; @@ -64,6 +65,7 @@ $application->add(new UpdateDataRetention()); $application->add(new OrganizationResetWorkspacePasswords()); $application->add(new ForceUnlinkSharedBuckets()); +$application->add(new ListExternalBuckets()); $application->add(new OrganizationsAddFeature()); $application->add(new DeleteProjects()); $application->add(new DeleteStorageBackend()); diff --git a/src/Keboola/Console/Command/ListExternalBuckets.php b/src/Keboola/Console/Command/ListExternalBuckets.php new file mode 100644 index 0000000..9f92d24 --- /dev/null +++ b/src/Keboola/Console/Command/ListExternalBuckets.php @@ -0,0 +1,143 @@ +setName('manage:list-external-buckets') + ->setDescription( + 'Read-only: list all external buckets (hasExternalSchema) across all projects of a stack ' + . 'with created date, isReadOnly flag and whether KBC.description metadata is set.' + ) + ->addArgument( + self::ARGUMENT_MANAGE_TOKEN, + InputArgument::REQUIRED, + 'Manage API token (super admin) used to create short-lived project storage tokens.' + ) + ->addArgument( + self::ARGUMENT_HOSTNAME_SUFFIX, + InputArgument::OPTIONAL, + 'Keboola Connection Hostname Suffix', + 'keboola.com' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $manageToken = $input->getArgument(self::ARGUMENT_MANAGE_TOKEN); + assert(is_string($manageToken)); + $hostnameSuffix = $input->getArgument(self::ARGUMENT_HOSTNAME_SUFFIX); + assert(is_string($hostnameSuffix)); + assert($hostnameSuffix !== ''); + + $serviceClient = new ServiceClient($hostnameSuffix); + $connectionUrl = $serviceClient->getConnectionServiceUrl(); + $manageClient = new ManageClient(['token' => $manageToken, 'url' => $connectionUrl]); + + $output->writeln('projectId,projectName,bucketId,created,isReadOnly,linked,hasDescription,description'); + + $externalBucketsFound = 0; + $projectsChecked = 0; + $projectsSkipped = 0; + $seenProjects = []; + + foreach ($manageClient->listMaintainers() as $maintainer) { + foreach ($manageClient->listMaintainerOrganizations($maintainer['id']) as $organization) { + foreach ($manageClient->listOrganizationProjects($organization['id']) as $project) { + $projectId = (int) $project['id']; + if (isset($seenProjects[$projectId])) { + continue; + } + $seenProjects[$projectId] = true; + + try { + $storageToken = $manageClient->createProjectStorageToken( + $projectId, + [ + 'description' => 'List external buckets (read-only audit)', + 'expiresIn' => 900, + 'canManageBuckets' => true, + ] + ); + } catch (\Throwable $e) { + $output->writeln(sprintf( + '# Access denied or error for project "%s" ("%s"): %s', + $projectId, + $project['name'], + $e->getMessage(), + )); + $projectsSkipped++; + continue; + } + assert(is_string($storageToken['token'])); + $projectsChecked++; + + $storageClient = new StorageApiClient([ + 'token' => $storageToken['token'], + 'url' => $connectionUrl, + ]); + + try { + $buckets = $storageClient->listBuckets(['include' => 'metadata']); + foreach ($buckets as $bucket) { + if (($bucket['hasExternalSchema'] ?? false) !== true) { + continue; + } + $externalBucketsFound++; + + $description = null; + foreach ($bucket['metadata'] ?? [] as $metadata) { + if ($metadata['key'] === 'KBC.description') { + $description = (string) $metadata['value']; + break; + } + } + + $output->writeln(sprintf( + '%d,"%s",%s,%s,%s,%s,%s,"%s"', + $projectId, + str_replace('"', '""', (string) $project['name']), + $bucket['id'], + $bucket['created'], + ($bucket['isReadOnly'] ?? false) ? 'READ_ONLY' : 'writable', + isset($bucket['sourceBucket']) ? 'linked' : 'own', + $description !== null ? 'yes' : 'no', + str_replace('"', '""', (string) ($description ?? '')), + )); + } + } finally { + $tokensClient = new Tokens($storageClient); + assert(is_scalar($storageToken['id'])); + $tokensClient->dropToken((int) $storageToken['id']); + } + } + } + } + + $output->writeln(sprintf( + '# Done: %d external buckets in %d projects checked (%d projects skipped).', + $externalBucketsFound, + $projectsChecked, + $projectsSkipped, + )); + + return 0; + } +}