diff --git a/ci-automation/release/azure_marketplace_publish.py b/ci-automation/release/azure_marketplace_publish.py index d0039eac8b7..5cef4cd1998 100644 --- a/ci-automation/release/azure_marketplace_publish.py +++ b/ci-automation/release/azure_marketplace_publish.py @@ -6,6 +6,7 @@ # ] # /// import os +import sys import copy import json import argparse @@ -14,15 +15,16 @@ from azure.storage.blob import BlobClient, generate_container_sas, BlobSasPermissions from datetime import datetime, timedelta +from urllib.parse import urlsplit, urlunsplit logging.basicConfig(level=logging.DEBUG) # Configuration data (previously in config.toml) CONFIG = { "az_storage": { - "account_name": "flatcar", + "account_name": "flatcar0001", "container_name": "publish", - "blob_name_format": "flatcar-linux-{version}-{plan}-{arch}.vhd" + "blob_name_format": "flatcar-linux-{version}-{plan}-{arch}.vhd", }, "offer_metadata": { "flatcar-container-linux-corevm": "arm64", @@ -31,34 +33,97 @@ "flatcar-container-linux": "amd64", }, "plan_metadata": { - "alpha": ["flatcar-container-linux-corevm", "flatcar-container-linux-corevm-amd64", "flatcar-container-linux-free", "flatcar-container-linux"], - "beta": ["flatcar-container-linux-corevm", "flatcar-container-linux-corevm-amd64", "flatcar-container-linux-free", "flatcar-container-linux"], - "stable": ["flatcar-container-linux-corevm", "flatcar-container-linux-corevm-amd64", "flatcar-container-linux-free", "flatcar-container-linux"], - "lts2024": ["flatcar-container-linux-free", "flatcar-container-linux", "flatcar-container-linux-corevm-amd64", "flatcar-container-linux-corevm"], + "alpha": [ + "flatcar-container-linux-corevm", + "flatcar-container-linux-corevm-amd64", + "flatcar-container-linux-free", + "flatcar-container-linux", + ], + "beta": [ + "flatcar-container-linux-corevm", + "flatcar-container-linux-corevm-amd64", + "flatcar-container-linux-free", + "flatcar-container-linux", + ], + "stable": [ + "flatcar-container-linux-corevm", + "flatcar-container-linux-corevm-amd64", + "flatcar-container-linux-free", + "flatcar-container-linux", + ], + "lts2024": [ + "flatcar-container-linux-free", + "flatcar-container-linux", + "flatcar-container-linux-corevm-amd64", + "flatcar-container-linux-corevm", + ], }, "test_offer_metadata": { "test-release-automation-corevm": "amd64", "test-release-automation": "amd64", }, "test_plan_metadata": { - "release-test-automation": ["test-release-automation-corevm", "test-release-automation"], - } + "release-test-automation": [ + "test-release-automation-corevm", + "test-release-automation", + ], + }, } -def get_active_plans(): - resp = requests.get('https://flatcar.cdn.cncf.io/channel-info.txt') +def get_channel_info(): + resp = requests.get("https://flatcar.cdn.cncf.io/channel-info.txt") if resp.status_code != 200: - logging.error("There is some issue with the channel-info.txt file. Please check https://flatcar.cdn.cncf.io/channel-info.txt") + logging.error( + "There is some issue with the channel-info.txt file. Please check https://flatcar.cdn.cncf.io/channel-info.txt" + ) logging.error(f"Returned status code: {resp.status_code}") + return {} + + entries = {} + for line in resp.text.strip().split("\n"): + key, _, value = line.partition("=") + entries[key.strip()] = value.strip() + + return entries - plans = [i.split("=")[0].replace('_CURRENT','').lower() for i in resp.text.strip().split('\n')] - plans = [plan for plan in plans if plan != 'lts'] + +def get_active_plans(channel_info): + plans = [key.replace("_CURRENT", "").lower() for key in channel_info] + plans = [plan for plan in plans if plan != "lts"] + # channel-info.txt uses "lts_" (e.g. "lts_2024"), but the plan + # names registered on Azure Marketplace drop the underscore, e.g. "lts2024". + plans = [plan.replace("lts_", "lts") for plan in plans] return plans +def resolve_lts_plan(channel_info, version): + """ + This function primarily to search the LTS version for the corresponding plans + """ + matches = [] + for key, value in channel_info.items(): + if ( + key == "LTS_CURRENT" + or not key.startswith("LTS_") + or not key.endswith("_CURRENT") + ): + continue + year = key[len("LTS_") : -len("_CURRENT")] + if year.isdigit() and value == version: + matches.append(year) + + if len(matches) != 1: + logging.error( + f"Could not LTS year for version {version} in channel-info.txt, matches: {matches}" + ) + return None + + return f"lts{matches[0]}" + + def generate_partner_center_token(tenant_id, client_id, secret_value): data = f"grant_type=client_credentials&client_id={client_id}&client_secret={secret_value}&resource=https://graph.microsoft.com" resp = requests.post( @@ -143,44 +208,73 @@ def get_image_versions(access_token, product_durable_id, plan_durable_id, corevm return resp.json().get("vmImageVersions") -def draft_new_image_versions( +def deprecate_oldest_image_version(image_versions, live_threshold=95): + """Mark the oldest non-deprecated entry in `image_versions` (as returned + by get_image_versions) as deprecated, in place, but only once the number + of live (non-deprecated) versions exceeds `live_threshold` (Azure caps a + SKU at 100 versions). Returns the deprecated entry, or None if nothing + was deprecated.""" + active_versions = [ + v for v in image_versions if v.get("lifecycleState") != "deprecated" + ] + if len(active_versions) <= live_threshold: + return None + + oldest = min( + active_versions, + key=lambda v: tuple(int(part) for part in v["versionNumber"].split(".")), + ) + oldest["lifecycleState"] = "deprecated" + return oldest + + +def image_version_exists(image_versions, version): + return any(v.get("versionNumber") == version for v in image_versions) + + +def delete_draft_image_version(image_versions, version): + """Mark the entry in `image_versions` matching `version` as deleted, in + place. Only draft entries that were never published can be deleted this + way (the API rejects deletion of anything already live). Returns the + deleted entry, or None if no entry matched `version`.""" + match = next((v for v in image_versions if v.get("versionNumber") == version), None) + if match is None: + return None + + match["lifecycleState"] = "deleted" + return match + + +def _redact_sas_uri(url): + """Strip the SAS query string (the actual credential) from a blob URL, keeping the path for debugging.""" + parsed = urlsplit(url) + if not parsed.query: + return url + return urlunsplit( + (parsed.scheme, parsed.netloc, parsed.path, "REDACTED", parsed.fragment) + ) + + +def _redact_payload_uris(payload): + payload = copy.deepcopy(payload) + for resource in payload.get("resources", []): + for image_version in resource.get("vmImageVersions", []): + for vm_image in image_version.get("vmImages", []): + os_disk = vm_image.get("source", {}).get("osDisk", {}) + if "uri" in os_disk: + os_disk["uri"] = _redact_sas_uri(os_disk["uri"]) + return payload + + +def submit_image_versions( access_token, plan, offer, - version, - az_sas_url, image_versions, image_type_arch, corevm=False, + dry_run=False, ): - new_vm_image = { - "versionNumber": version, - "vmImages": [ - { - "imageType": f"{image_type_arch}Gen2", - "source": { - "sourceType": "sasUri", - "osDisk": {"uri": az_sas_url}, - "dataDisks": [], - }, - }, - ], - } - - if image_type_arch != "arm64": - new_vm_image["vmImages"].append( - { - "imageType": f"{image_type_arch}Gen1", - "source": { - "sourceType": "sasUri", - "osDisk": {"uri": az_sas_url}, - "dataDisks": [], - }, - } - ) - - image_versions.append(new_vm_image) - schema_url = "https://schema.mp.microsoft.com/schema/virtual-machine-plan-technical-configuration/2022-03-01-preview3" if corevm: schema_url = "https://schema.mp.microsoft.com/schema/core-virtual-machine-plan-technical-configuration/2022-03-01-preview5" @@ -244,6 +338,11 @@ def draft_new_image_versions( if corevm: payload["resources"][0]["softwareType"] = "operatingSystem" + if dry_run: + print(f"[dry-run] Would POST configure for {offer}/{plan}:") + print(json.dumps(_redact_payload_uris(payload), indent=2)) + return True + resp = requests.post( url=f"https://graph.microsoft.com/rp/product-ingestion/configure", headers={ @@ -254,25 +353,143 @@ def draft_new_image_versions( data=json.dumps(payload), ) + if resp.status_code != 202: + logging.error( + f"submit_image_versions failed for {offer}/{plan}: " + f"status={resp.status_code} body={resp.text}" + ) + return False + + return True + + +def draft_new_image_versions( + access_token, + plan, + offer, + version, + az_sas_url, + image_versions, + image_type_arch, + corevm=False, + dry_run=False, +): + new_vm_image = { + "versionNumber": version, + "vmImages": [ + { + "imageType": f"{image_type_arch}Gen2", + "source": { + "sourceType": "sasUri", + "osDisk": {"uri": az_sas_url}, + "dataDisks": [], + }, + }, + ], + } + + if image_type_arch != "arm64": + new_vm_image["vmImages"].append( + { + "imageType": f"{image_type_arch}Gen1", + "source": { + "sourceType": "sasUri", + "osDisk": {"uri": az_sas_url}, + "dataDisks": [], + }, + } + ) + + image_versions.append(new_vm_image) + + return submit_image_versions( + access_token, + plan, + offer, + image_versions, + image_type_arch, + corevm=corevm, + dry_run=dry_run, + ) + def main(): parser = argparse.ArgumentParser( prog="azure-marketlace-ingestion-api", description="Program to publish the Azure Marketplace Images", ) - parser.add_argument("-p", "--plan") - parser.add_argument("-v", "--version") - parser.add_argument("-s", "--az-sas-url") - parser.add_argument("-t", "--test-mode", action="store_true") - parser.add_argument("-z", "--test-plan") + parser.add_argument( + "-p", + "--plan", + help="Channel/plan to publish, e.g. alpha, beta, stable, lts, lts2024", + ) + parser.add_argument( + "-v", + "--version", + help="Flatcar version to publish, e.g. 4081.3.10. Required unless --deprecate-only is set", + ) + parser.add_argument( + "-s", + "--az-sas-url", + help="SAS URL of the VHD blob to publish; generated automatically when omitted", + ) + parser.add_argument( + "-t", + "--test-mode", + action="store_true", + help="Publish to the test offer/plan instead of the real ones", + ) + parser.add_argument( + "-z", + "--test-plan", + help="Plan name to use for the blob lookup in test mode", + ) + parser.add_argument( + "-d", + "--deprecate-only", + action="store_true", + help="Deprecate the oldest image version for the plan's offers and exit, without publishing a new version", + ) + parser.add_argument( + "-x", + "--delete-draft-version", + action="store_true", + help="Delete the draft (not-yet-published) image version given by --version, across the plan's offers, and exit", + ) + parser.add_argument( + "-n", + "--dry-run", + action="store_true", + help="Perform all read-only lookups but skip the actual submit/publish API calls, printing what would be sent instead", + ) args = parser.parse_args() - if not all((args.plan, args.version)): - logging.error("Both version and plan is required") + if not args.plan or ( + not args.deprecate_only and not args.delete_draft_version and not args.version + ): + logging.error( + "plan is required, and version is required unless --deprecate-only or --delete-draft-version is set" + ) + return + + if args.delete_draft_version and not args.version: + logging.error("--version is required when --delete-draft-version is set") return - active_plans = get_active_plans() + channel_info = get_channel_info() + active_plans = get_active_plans(channel_info) plan = args.plan + if not args.test_mode and plan == "lts": + lts_version = args.version or channel_info.get("LTS_CURRENT") + if not lts_version: + logging.error( + "--plan lts requires --version, or channel-info.txt must contain LTS_CURRENT" + ) + return + plan = resolve_lts_plan(channel_info, lts_version) + if plan is None: + return + if not args.test_mode and plan not in active_plans: logging.error(f"plan value should be either {', '.join(active_plans)}") return @@ -297,16 +514,12 @@ def main(): if args.test_mode: OFFER_METADATA = CONFIG.get("test_offer_metadata") if not OFFER_METADATA: - logging.error( - "test_mode: Missing `test_offer_metadata` section in config" - ) + logging.error("test_mode: Missing `test_offer_metadata` section in config") return PLAN_METADATA = CONFIG.get("test_plan_metadata") if not PLAN_METADATA: - logging.error( - "test_mode: Missing `test_plan_metadata` section in config" - ) + logging.error("test_mode: Missing `test_plan_metadata` section in config") return else: OFFER_METADATA = CONFIG.get("offer_metadata") @@ -317,11 +530,8 @@ def main(): if not PLAN_METADATA: logging.error("Missing `plan_metadata` section in config") + failed_offers = [] for offer in PLAN_METADATA.get(plan, []): - az_sas_url = None - if args.az_sas_url is not None: - az_sas_url = args.az_sas_url - corevm = False if "corevm" in offer: corevm = True @@ -330,16 +540,21 @@ def main(): if arch is None: continue - if az_sas_url is None: - kwargs = {} - if test_plan: - kwargs = {"test_plan": test_plan} - az_sas_url = generate_az_sas_url("lts" if plan.startswith("lts") else plan, version, arch, **kwargs) + az_sas_url = None + if not args.deprecate_only and not args.delete_draft_version: + az_sas_url = args.az_sas_url if az_sas_url is None: - logging.error( - f"generate_az_sas_url returned None for {plan}, {version}, {arch}" + kwargs = {} + if test_plan: + kwargs = {"test_plan": test_plan} + az_sas_url = generate_az_sas_url( + "lts" if plan.startswith("lts") else plan, version, arch, **kwargs ) - continue + if az_sas_url is None: + logging.error( + f"generate_az_sas_url returned None for {plan}, {version}, {arch}" + ) + continue product_durable_id = get_product_durable_id(access_token, offer) plan_durable_id = get_plan_durable_id(access_token, product_durable_id, plan) @@ -355,7 +570,56 @@ def main(): if OFFER_METADATA[offer] == "arm64": image_type_arch = "arm64" - draft_new_image_versions( + if args.delete_draft_version: + deleted = delete_draft_image_version(image_versions, version) + if deleted is None: + print(f"No draft version {version} found for {offer}") + continue + success = submit_image_versions( + access_token, + plan, + offer, + image_versions, + image_type_arch, + corevm=corevm, + dry_run=args.dry_run, + ) + if not success: + failed_offers.append(offer) + continue + verb = "Would delete" if args.dry_run else "Deleted" + print(f"{verb} draft image version {version} for {offer}") + continue + + if not args.deprecate_only and image_version_exists(image_versions, version): + print(f"Version {version} already exists for {offer}, skipping") + continue + + deprecated = deprecate_oldest_image_version(image_versions) + if deprecated is not None: + verb = "Would deprecate" if args.dry_run else "Deprecating" + print( + f"{verb} oldest image version {deprecated['versionNumber']} for {offer}" + ) + + if args.deprecate_only: + if deprecated is None: + print(f"Nothing to deprecate for {offer}") + continue + success = submit_image_versions( + access_token, + plan, + offer, + image_versions, + image_type_arch, + corevm=corevm, + dry_run=args.dry_run, + ) + if not success: + failed_offers.append(offer) + continue + + success = draft_new_image_versions( access_token, plan, offer, @@ -364,8 +628,19 @@ def main(): image_versions, image_type_arch, corevm=corevm, + dry_run=args.dry_run, ) - print("Done preparing offers, you now have to click the publish button for each offer in https://partner.microsoft.com/en-us/dashboard/marketplace-offers/overview") + if not success: + failed_offers.append(offer) + continue + if not args.dry_run: + print( + f"Done preparing {offer}, you now have to click the publish button for it in https://partner.microsoft.com/en-us/dashboard/marketplace-offers/overview" + ) + + if failed_offers: + logging.error(f"Submission failed for offers: {', '.join(failed_offers)}") + sys.exit(1) if __name__ == "__main__":