From db54ddfae31d2ddcfc9517498e500ed5e371dabf Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Tue, 18 Aug 2026 20:34:31 +0530 Subject: [PATCH] Add better handling for the azure marketplace updates This commit fixes: - Add better parsing for the LTS, earlier this was broken and the LTS release was not found in the list of supported channels. - Update the account_name to flatcar0001 - We hitting the 100 limit for the Azure images no on a regular basis so added a threshold to remove the images when the number is higher than 95 - Add the option to delete the draft versions which is quite difficult to do manually as it involves time and effort and the script works a bit differently where it handles channel across plans where if any thing needs to rectified then all the plans needs to be visited and handled manually. The commit eases the process a bit - Add a dry run mode to check if everything will be well run - Formatting using black Signed-off-by: Sayan Chowdhury --- .../release/azure_marketplace_publish.py | 393 ++++++++++++++---- 1 file changed, 321 insertions(+), 72 deletions(-) diff --git a/ci-automation/release/azure_marketplace_publish.py b/ci-automation/release/azure_marketplace_publish.py index d0039eac8b7..e9c7bf6d99d 100644 --- a/ci-automation/release/azure_marketplace_publish.py +++ b/ci-automation/release/azure_marketplace_publish.py @@ -6,6 +6,7 @@ # ] # /// import os +import sys import copy import json import argparse @@ -20,9 +21,9 @@ # Configuration data (previously in config.toml) CONFIG = { "az_storage": { - "account_name": "flatcar", + "account_name": "flatcar0001", "container_name": "publish", - "blob_name_format": "flatcar-linux-{version}-{plan}-{arch}.vhd" + "blob_name_format": "flatcar-linux-{version}-{plan}-{arch}.vhd", }, "offer_metadata": { "flatcar-container-linux-corevm": "arm64", @@ -31,34 +32,97 @@ "flatcar-container-linux": "amd64", }, "plan_metadata": { - "alpha": ["flatcar-container-linux-corevm", "flatcar-container-linux-corevm-amd64", "flatcar-container-linux-free", "flatcar-container-linux"], - "beta": ["flatcar-container-linux-corevm", "flatcar-container-linux-corevm-amd64", "flatcar-container-linux-free", "flatcar-container-linux"], - "stable": ["flatcar-container-linux-corevm", "flatcar-container-linux-corevm-amd64", "flatcar-container-linux-free", "flatcar-container-linux"], - "lts2024": ["flatcar-container-linux-free", "flatcar-container-linux", "flatcar-container-linux-corevm-amd64", "flatcar-container-linux-corevm"], + "alpha": [ + "flatcar-container-linux-corevm", + "flatcar-container-linux-corevm-amd64", + "flatcar-container-linux-free", + "flatcar-container-linux", + ], + "beta": [ + "flatcar-container-linux-corevm", + "flatcar-container-linux-corevm-amd64", + "flatcar-container-linux-free", + "flatcar-container-linux", + ], + "stable": [ + "flatcar-container-linux-corevm", + "flatcar-container-linux-corevm-amd64", + "flatcar-container-linux-free", + "flatcar-container-linux", + ], + "lts2024": [ + "flatcar-container-linux-free", + "flatcar-container-linux", + "flatcar-container-linux-corevm-amd64", + "flatcar-container-linux-corevm", + ], }, "test_offer_metadata": { "test-release-automation-corevm": "amd64", "test-release-automation": "amd64", }, "test_plan_metadata": { - "release-test-automation": ["test-release-automation-corevm", "test-release-automation"], - } + "release-test-automation": [ + "test-release-automation-corevm", + "test-release-automation", + ], + }, } -def get_active_plans(): - resp = requests.get('https://flatcar.cdn.cncf.io/channel-info.txt') +def get_channel_info(): + resp = requests.get("https://flatcar.cdn.cncf.io/channel-info.txt") if resp.status_code != 200: - logging.error("There is some issue with the channel-info.txt file. Please check https://flatcar.cdn.cncf.io/channel-info.txt") + logging.error( + "There is some issue with the channel-info.txt file. Please check https://flatcar.cdn.cncf.io/channel-info.txt" + ) logging.error(f"Returned status code: {resp.status_code}") + return {} + + entries = {} + for line in resp.text.strip().split("\n"): + key, _, value = line.partition("=") + entries[key.strip()] = value.strip() - plans = [i.split("=")[0].replace('_CURRENT','').lower() for i in resp.text.strip().split('\n')] - plans = [plan for plan in plans if plan != 'lts'] + return entries + + +def get_active_plans(channel_info): + plans = [key.replace("_CURRENT", "").lower() for key in channel_info] + plans = [plan for plan in plans if plan != "lts"] + # channel-info.txt uses "lts_" (e.g. "lts_2024"), but the plan + # names registered on Azure Marketplace drop the underscore, e.g. "lts2024". + plans = [plan.replace("lts_", "lts") for plan in plans] return plans +def resolve_lts_plan(channel_info, version): + """ + This function primarily to search the LTS version for the corresponding plans + """ + matches = [] + for key, value in channel_info.items(): + if ( + key == "LTS_CURRENT" + or not key.startswith("LTS_") + or not key.endswith("_CURRENT") + ): + continue + year = key[len("LTS_") : -len("_CURRENT")] + if year.isdigit() and value == version: + matches.append(year) + + if len(matches) != 1: + logging.error( + f"Could not LTS year for version {version} in channel-info.txt, matches: {matches}" + ) + return None + + return f"lts{matches[0]}" + + def generate_partner_center_token(tenant_id, client_id, secret_value): data = f"grant_type=client_credentials&client_id={client_id}&client_secret={secret_value}&resource=https://graph.microsoft.com" resp = requests.post( @@ -143,44 +207,54 @@ def get_image_versions(access_token, product_durable_id, plan_durable_id, corevm return resp.json().get("vmImageVersions") -def draft_new_image_versions( +def deprecate_oldest_image_version(image_versions, live_threshold=95): + """Mark the oldest non-deprecated entry in `image_versions` (as returned + by get_image_versions) as deprecated, in place, but only once the number + of live (non-deprecated) versions exceeds `live_threshold` (Azure caps a + SKU at 100 versions). Returns the deprecated entry, or None if nothing + was deprecated.""" + active_versions = [ + v for v in image_versions if v.get("lifecycleState") != "deprecated" + ] + if len(active_versions) <= live_threshold: + return None + + oldest = min( + active_versions, + key=lambda v: tuple(int(part) for part in v["versionNumber"].split(".")), + ) + oldest["lifecycleState"] = "deprecated" + return oldest + + +def image_version_exists(image_versions, version): + return any(v.get("versionNumber") == version for v in image_versions) + + +def delete_draft_image_version(image_versions, version): + """Mark the entry in `image_versions` matching `version` as deleted, in + place. Only draft entries that were never published can be deleted this + way (the API rejects deletion of anything already live). Returns the + deleted entry, or None if no entry matched `version`.""" + match = next( + (v for v in image_versions if v.get("versionNumber") == version), None + ) + if match is None: + return None + + match["lifecycleState"] = "deleted" + return match + + +def submit_image_versions( access_token, plan, offer, - version, - az_sas_url, image_versions, image_type_arch, corevm=False, + dry_run=False, ): - new_vm_image = { - "versionNumber": version, - "vmImages": [ - { - "imageType": f"{image_type_arch}Gen2", - "source": { - "sourceType": "sasUri", - "osDisk": {"uri": az_sas_url}, - "dataDisks": [], - }, - }, - ], - } - - if image_type_arch != "arm64": - new_vm_image["vmImages"].append( - { - "imageType": f"{image_type_arch}Gen1", - "source": { - "sourceType": "sasUri", - "osDisk": {"uri": az_sas_url}, - "dataDisks": [], - }, - } - ) - - image_versions.append(new_vm_image) - schema_url = "https://schema.mp.microsoft.com/schema/virtual-machine-plan-technical-configuration/2022-03-01-preview3" if corevm: schema_url = "https://schema.mp.microsoft.com/schema/core-virtual-machine-plan-technical-configuration/2022-03-01-preview5" @@ -244,6 +318,11 @@ def draft_new_image_versions( if corevm: payload["resources"][0]["softwareType"] = "operatingSystem" + if dry_run: + print(f"[dry-run] Would POST configure for {offer}/{plan}:") + print(json.dumps(payload, indent=2)) + return True + resp = requests.post( url=f"https://graph.microsoft.com/rp/product-ingestion/configure", headers={ @@ -254,25 +333,137 @@ def draft_new_image_versions( data=json.dumps(payload), ) + if resp.status_code != 202: + logging.error( + f"submit_image_versions failed for {offer}/{plan}: " + f"status={resp.status_code} body={resp.text}" + ) + return False + + return True + + +def draft_new_image_versions( + access_token, + plan, + offer, + version, + az_sas_url, + image_versions, + image_type_arch, + corevm=False, + dry_run=False, +): + new_vm_image = { + "versionNumber": version, + "vmImages": [ + { + "imageType": f"{image_type_arch}Gen2", + "source": { + "sourceType": "sasUri", + "osDisk": {"uri": az_sas_url}, + "dataDisks": [], + }, + }, + ], + } + + if image_type_arch != "arm64": + new_vm_image["vmImages"].append( + { + "imageType": f"{image_type_arch}Gen1", + "source": { + "sourceType": "sasUri", + "osDisk": {"uri": az_sas_url}, + "dataDisks": [], + }, + } + ) + + image_versions.append(new_vm_image) + + return submit_image_versions( + access_token, + plan, + offer, + image_versions, + image_type_arch, + corevm=corevm, + dry_run=dry_run, + ) + def main(): parser = argparse.ArgumentParser( prog="azure-marketlace-ingestion-api", description="Program to publish the Azure Marketplace Images", ) - parser.add_argument("-p", "--plan") - parser.add_argument("-v", "--version") - parser.add_argument("-s", "--az-sas-url") - parser.add_argument("-t", "--test-mode", action="store_true") - parser.add_argument("-z", "--test-plan") + parser.add_argument( + "-p", + "--plan", + help="Channel/plan to publish, e.g. alpha, beta, stable, lts, lts2024", + ) + parser.add_argument( + "-v", + "--version", + help="Flatcar version to publish, e.g. 4081.3.10. Required unless --deprecate-only is set", + ) + parser.add_argument( + "-s", + "--az-sas-url", + help="SAS URL of the VHD blob to publish; generated automatically when omitted", + ) + parser.add_argument( + "-t", + "--test-mode", + action="store_true", + help="Publish to the test offer/plan instead of the real ones", + ) + parser.add_argument( + "-z", + "--test-plan", + help="Plan name to use for the blob lookup in test mode", + ) + parser.add_argument( + "-d", + "--deprecate-only", + action="store_true", + help="Deprecate the oldest image version for the plan's offers and exit, without publishing a new version", + ) + parser.add_argument( + "-x", + "--delete-draft-version", + action="store_true", + help="Delete the draft (not-yet-published) image version given by --version, across the plan's offers, and exit", + ) + parser.add_argument( + "-n", + "--dry-run", + action="store_true", + help="Perform all read-only lookups but skip the actual submit/publish API calls, printing what would be sent instead", + ) args = parser.parse_args() - if not all((args.plan, args.version)): - logging.error("Both version and plan is required") + if not args.plan or ( + not args.deprecate_only and not args.delete_draft_version and not args.version + ): + logging.error( + "plan is required, and version is required unless --deprecate-only or --delete-draft-version is set" + ) + return + + if args.delete_draft_version and not args.version: + logging.error("--version is required when --delete-draft-version is set") return - active_plans = get_active_plans() + channel_info = get_channel_info() + active_plans = get_active_plans(channel_info) plan = args.plan + if not args.test_mode and plan == "lts": + plan = resolve_lts_plan(channel_info, args.version) + if plan is None: + return + if not args.test_mode and plan not in active_plans: logging.error(f"plan value should be either {', '.join(active_plans)}") return @@ -297,16 +488,12 @@ def main(): if args.test_mode: OFFER_METADATA = CONFIG.get("test_offer_metadata") if not OFFER_METADATA: - logging.error( - "test_mode: Missing `test_offer_metadata` section in config" - ) + logging.error("test_mode: Missing `test_offer_metadata` section in config") return PLAN_METADATA = CONFIG.get("test_plan_metadata") if not PLAN_METADATA: - logging.error( - "test_mode: Missing `test_plan_metadata` section in config" - ) + logging.error("test_mode: Missing `test_plan_metadata` section in config") return else: OFFER_METADATA = CONFIG.get("offer_metadata") @@ -317,11 +504,8 @@ def main(): if not PLAN_METADATA: logging.error("Missing `plan_metadata` section in config") + failed_offers = [] for offer in PLAN_METADATA.get(plan, []): - az_sas_url = None - if args.az_sas_url is not None: - az_sas_url = args.az_sas_url - corevm = False if "corevm" in offer: corevm = True @@ -330,16 +514,21 @@ def main(): if arch is None: continue - if az_sas_url is None: - kwargs = {} - if test_plan: - kwargs = {"test_plan": test_plan} - az_sas_url = generate_az_sas_url("lts" if plan.startswith("lts") else plan, version, arch, **kwargs) + az_sas_url = None + if not args.deprecate_only and not args.delete_draft_version: + az_sas_url = args.az_sas_url if az_sas_url is None: - logging.error( - f"generate_az_sas_url returned None for {plan}, {version}, {arch}" + kwargs = {} + if test_plan: + kwargs = {"test_plan": test_plan} + az_sas_url = generate_az_sas_url( + "lts" if plan.startswith("lts") else plan, version, arch, **kwargs ) - continue + if az_sas_url is None: + logging.error( + f"generate_az_sas_url returned None for {plan}, {version}, {arch}" + ) + continue product_durable_id = get_product_durable_id(access_token, offer) plan_durable_id = get_plan_durable_id(access_token, product_durable_id, plan) @@ -355,7 +544,56 @@ def main(): if OFFER_METADATA[offer] == "arm64": image_type_arch = "arm64" - draft_new_image_versions( + if args.delete_draft_version: + deleted = delete_draft_image_version(image_versions, version) + if deleted is None: + print(f"No draft version {version} found for {offer}") + continue + success = submit_image_versions( + access_token, + plan, + offer, + image_versions, + image_type_arch, + corevm=corevm, + dry_run=args.dry_run, + ) + if not success: + failed_offers.append(offer) + continue + verb = "Would delete" if args.dry_run else "Deleted" + print(f"{verb} draft image version {version} for {offer}") + continue + + if not args.deprecate_only and image_version_exists(image_versions, version): + print(f"Version {version} already exists for {offer}, skipping") + continue + + deprecated = deprecate_oldest_image_version(image_versions) + if deprecated is not None: + verb = "Would deprecate" if args.dry_run else "Deprecating" + print( + f"{verb} oldest image version {deprecated['versionNumber']} for {offer}" + ) + + if args.deprecate_only: + if deprecated is None: + print(f"Nothing to deprecate for {offer}") + continue + success = submit_image_versions( + access_token, + plan, + offer, + image_versions, + image_type_arch, + corevm=corevm, + dry_run=args.dry_run, + ) + if not success: + failed_offers.append(offer) + continue + + success = draft_new_image_versions( access_token, plan, offer, @@ -364,8 +602,19 @@ def main(): image_versions, image_type_arch, corevm=corevm, + dry_run=args.dry_run, ) - print("Done preparing offers, you now have to click the publish button for each offer in https://partner.microsoft.com/en-us/dashboard/marketplace-offers/overview") + if not success: + failed_offers.append(offer) + continue + if not args.dry_run: + print( + f"Done preparing {offer}, you now have to click the publish button for it in https://partner.microsoft.com/en-us/dashboard/marketplace-offers/overview" + ) + + if failed_offers: + logging.error(f"Submission failed for offers: {', '.join(failed_offers)}") + sys.exit(1) if __name__ == "__main__":