From 2c2d5989a04bb01d9893538bf9559a1e8b9781f9 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 13:54:55 -0400 Subject: [PATCH 01/10] RELOPS-2548: Add FooFrix Azure subscription and Key Vault --- terraform/azure_ad/foofrix.tf | 13 +++ terraform/azure_foofrix/README.md | 108 ++++++++++++++++++++++ terraform/azure_foofrix/access.tftest.hcl | 47 ++++++++++ terraform/azure_foofrix/backend.tf | 8 ++ terraform/azure_foofrix/keyvault.tf | 44 +++++++++ terraform/azure_foofrix/main.tf | 87 +++++++++++++++++ terraform/azure_foofrix/outputs.tf | 19 ++++ terraform/azure_foofrix/providers.tf | 33 +++++++ 8 files changed, 359 insertions(+) create mode 100644 terraform/azure_ad/foofrix.tf create mode 100644 terraform/azure_foofrix/README.md create mode 100644 terraform/azure_foofrix/access.tftest.hcl create mode 100644 terraform/azure_foofrix/backend.tf create mode 100644 terraform/azure_foofrix/keyvault.tf create mode 100644 terraform/azure_foofrix/main.tf create mode 100644 terraform/azure_foofrix/outputs.tf create mode 100644 terraform/azure_foofrix/providers.tf diff --git a/terraform/azure_ad/foofrix.tf b/terraform/azure_ad/foofrix.tf new file mode 100644 index 00000000..06c2ce31 --- /dev/null +++ b/terraform/azure_ad/foofrix.tf @@ -0,0 +1,13 @@ +# FooFrix identities. Subscription roles are in azure_foofrix. RELOPS-2548. +# Add GCP federation when the service account unique ID is known. +resource "azuread_application" "foofrix" { + display_name = "sp-foofrix-azure-devtest" + owners = data.azuread_group.relops.members + notes = "GCP provisioning identity for the FooFrix Azure DevTest Subscription. RELOPS-2548." +} + +resource "azuread_service_principal" "foofrix" { + client_id = azuread_application.foofrix.client_id + owners = data.azuread_group.relops.members + tags = concat(["name:sp-foofrix-azure-devtest"], local.sp_tags) +} diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md new file mode 100644 index 00000000..17b1de09 --- /dev/null +++ b/terraform/azure_foofrix/README.md @@ -0,0 +1,108 @@ +# FooFrix Azure subscription + +This stack implements the subscription plan in the +[September 11 meeting notes](https://mozilla-hub.atlassian.net/browse/RELOPS-2548?focusedCommentId=1727342). +It follows `azure_fuzzing`: `azure_ad` owns the application and service principal; +this stack owns the subscription and Azure resources. + +The draft uses Central US and gives Denis Palmeiro access. Confirm the region +and full Perf access list before deployment. Perf will create and remove its +VMs. This stack does not create a Taskcluster pool. + +| Identity | Access | +| --- | --- | +| Relops group | Subscription Owner; Key Vault Administrator | +| Perf members in `main.tf` | Subscription Contributor; Key Vault Secrets Officer | +| `sp-foofrix-azure-devtest` | Contributor on the FooFrix subscription | +| `id-foofrix-worker` | Read secrets in the FooFrix vault | + +Attach `id-foofrix-worker` to each FooFrix VM. Use its client ID to select it +when the agent reads Key Vault. Perf members can add AI keys through Key Vault. +Keep secret values out of Terraform, VM images, and startup scripts. Contributor +access lets the provisioner attach this identity without permission to create +role assignments. It therefore also permits indirect access to these secrets +through a VM that it controls. + +## Initial deployment + +The subscription must exist before the default Azure provider can use it. +Use two stages for the first deployment. The `billing` provider uses the existing +FXCI subscription only to call the subscription creation API. + +1. In `azure_ad`, review and apply the FooFrix application and service + principal. It has no credential yet. Add GCP federation after the service + account unique ID is known. +2. In this directory, initialize the backend and review the subscription plan: + + ```sh + export AWS_PROFILE=AdministratorAccess-961225894672 + terraform init + terraform plan -target=azurerm_subscription.foofrix -out=subscription.tfplan + ``` + +3. Apply the reviewed subscription plan. Then review a full plan: + + ```sh + terraform apply subscription.tfplan + terraform plan -out=foofrix.tfplan + ``` + +4. Apply the reviewed full plan. Use full plans for later changes. + Give Perf the subscription ID, provisioner client ID, worker identity ID, + worker identity client ID, and vault URI from `terraform output`. + +## GCP authentication + +For GCP to Azure, add a federated credential to the FooFrix application once +Perf supplies the GCP service account's numeric unique ID. Use issuer +`https://accounts.google.com` and audience `api://AzureADTokenExchange`. +See the [Microsoft GCP federation guide](https://learn.microsoft.com/entra/workload-id/workload-identity-federation-google-cloud). + +For Azure to GCS, use the worker managed identity with Google Workload Identity +Federation. This still needs the GCP project, results bucket, and required object +operations. Configure the Entra audience application, Google trust provider, +identity restriction, and bucket access after those values are known. These +resources are not part of this draft. See the +[Google Azure federation guide](https://docs.cloud.google.com/iam/docs/workload-identity-federation-with-other-clouds). + +## Windows VM and image work + +Start with one regular VM. Do not configure Spot eviction or a one-hour shutdown. +The first run must last at least 24 hours and complete a Firefox build and a +FooFrix test. Perf can then increase the count to two or three. + +For a GPU proof of concept, evaluate `Standard_NV18ads_A10_v5` (18 vCPUs, +220 GiB RAM, half an A10 GPU). If the test needs a full GPU, evaluate +`Standard_NV36ads_A10_v5` (36 vCPUs, 440 GiB RAM, one A10). Confirm regional +availability and quota in the new subscription. These are candidates, pending +the harness requirements. See the [Azure size table](https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/gpu-accelerated/nvadsa10v5-series). +Start with a 1 TiB persistent build disk and measure peak use. Temporary storage +must not hold the only copy of source changes or results. + +Provisioning needs Azure CLI or an Azure SDK, a VNet and subnet, a restricted +remote access rule, a Windows image version, persistent disks, and the worker +identity. Select the exact image and access method after repository inspection. + +`worker-images` has the Packer and Azure Compute Gallery build path. Its Windows +configs select Puppet roles and Pester tests. Its GitHub workflows also check +`.github/relsre.json`; repository access alone does not permit a build. + +Before adding a FooFrix image, inspect the harness to determine whether it needs +a prebuilt Chromium release or a Chromium source build with release options. +Confirm the version, build flags, toolchain, GPU driver, expected paths, and +update process. Add a separate image config and checks for Firefox builds, +Chromium startup, GPU use, and long VM lifetime. Confirm that Taskcluster startup +and shutdown services cannot terminate the standalone VM. Arrange gallery read +access from the FooFrix subscription and build access for the named Perf users. + +Any later use in a production Firefox CI pool must pass all tier 1 tasks from +the latest autoland decision task. A new tier 1 regression blocks deployment. + +## Checks + +```sh +terraform init -backend=false +terraform fmt -check +terraform validate +terraform test +``` diff --git a/terraform/azure_foofrix/access.tftest.hcl b/terraform/azure_foofrix/access.tftest.hcl new file mode 100644 index 00000000..7efa047d --- /dev/null +++ b/terraform/azure_foofrix/access.tftest.hcl @@ -0,0 +1,47 @@ +mock_provider "azurerm" { + mock_resource "azurerm_key_vault" { + defaults = { + id = "/subscriptions/11111111-2222-3333-4444-555555555555/resourceGroups/rg-foofrix/providers/Microsoft.KeyVault/vaults/kv-foofrix-test" + } + } +} +mock_provider "azurerm" { + alias = "billing" +} +mock_provider "azuread" {} + +override_resource { + target = azurerm_subscription.foofrix + values = { + subscription_id = "11111111-2222-3333-4444-555555555555" + } +} + +run "access_boundaries" { + # All providers are mocked; this does not create cloud resources. + command = apply + + assert { + condition = ( + azurerm_role_assignment.foofrix_contributor.scope == "/subscriptions/11111111-2222-3333-4444-555555555555" && + azurerm_role_assignment.foofrix_contributor.role_definition_name == "Contributor" + ) + error_message = "The GCP provisioner must have Contributor access only in FooFrix." + } + + assert { + condition = ( + azurerm_role_assignment.worker_secrets_user.scope == azurerm_key_vault.foofrix.id && + azurerm_role_assignment.worker_secrets_user.role_definition_name == "Key Vault Secrets User" + ) + error_message = "The worker must have read access at the FooFrix vault scope." + } + + assert { + condition = ( + azurerm_role_assignment.relops_owner.scope == "/subscriptions/11111111-2222-3333-4444-555555555555" && + azurerm_role_assignment.relops_owner.role_definition_name == "Owner" + ) + error_message = "RelOps must retain ownership of the new subscription." + } +} diff --git a/terraform/azure_foofrix/backend.tf b/terraform/azure_foofrix/backend.tf new file mode 100644 index 00000000..c7e487b8 --- /dev/null +++ b/terraform/azure_foofrix/backend.tf @@ -0,0 +1,8 @@ +terraform { + backend "s3" { + bucket = "relops-tf-states" + key = "azure_foofrix.tfstate" + use_lockfile = true + region = "us-west-2" + } +} diff --git a/terraform/azure_foofrix/keyvault.tf b/terraform/azure_foofrix/keyvault.tf new file mode 100644 index 00000000..70eb16cc --- /dev/null +++ b/terraform/azure_foofrix/keyvault.tf @@ -0,0 +1,44 @@ +resource "azurerm_user_assigned_identity" "worker" { + name = "id-foofrix-worker" + resource_group_name = azurerm_resource_group.foofrix.name + location = local.location + tags = local.common_tags + + depends_on = [azurerm_resource_provider_registration.this["Microsoft.ManagedIdentity"]] +} + +resource "azurerm_key_vault" "foofrix" { + name = "kv-foofrix-${substr(azurerm_subscription.foofrix.subscription_id, 0, 8)}" + resource_group_name = azurerm_resource_group.foofrix.name + location = local.location + tenant_id = local.tenant_id + sku_name = "standard" + rbac_authorization_enabled = true + purge_protection_enabled = true + tags = local.common_tags + + depends_on = [azurerm_resource_provider_registration.this["Microsoft.KeyVault"]] +} + +resource "azurerm_role_assignment" "relops_key_vault_administrator" { + scope = azurerm_key_vault.foofrix.id + role_definition_name = "Key Vault Administrator" + principal_id = data.azuread_group.relops.object_id + principal_type = "Group" +} + +resource "azurerm_role_assignment" "perf_secrets_officer" { + for_each = data.azuread_user.perf + scope = azurerm_key_vault.foofrix.id + role_definition_name = "Key Vault Secrets Officer" + principal_id = each.value.object_id + principal_type = "User" +} + +resource "azurerm_role_assignment" "worker_secrets_user" { + scope = azurerm_key_vault.foofrix.id + role_definition_name = "Key Vault Secrets User" + principal_id = azurerm_user_assigned_identity.worker.principal_id + principal_type = "ServicePrincipal" + skip_service_principal_aad_check = true +} diff --git a/terraform/azure_foofrix/main.tf b/terraform/azure_foofrix/main.tf new file mode 100644 index 00000000..bbb01826 --- /dev/null +++ b/terraform/azure_foofrix/main.tf @@ -0,0 +1,87 @@ +locals { + tenant_id = "c0dc8bb0-b616-427e-8217-9513964a145b" + location = "centralus" + + billing_account_id = "05ef9068-c74c-54a9-5b8f-82f7fb8b32cd:6e104178-9e3c-470c-9787-8ef53f372665_2019-05-31" + mozilla_billing_profile_id = "GRUW-TLBL-BG7-PGB" + mozilla_invoice_section_id = "VVEC-AWWS-PJA-PGB" + billing_scope_id = "/providers/Microsoft.Billing/billingAccounts/${local.billing_account_id}/billingProfiles/${local.mozilla_billing_profile_id}/invoiceSections/${local.mozilla_invoice_section_id}" + + # Confirm the complete Perf access list before deployment. + perf_members = toset(["dpalmeiro@mozilla.com"]) + + common_tags = { + terraform = "true" + project_name = "azure_foofrix" + production_state = "production" + owner_email = "relops@mozilla.com" + source_repo_url = "https://github.com/mozilla-platform-ops/relops_infra_as_code" + } +} + +resource "azurerm_subscription" "foofrix" { + provider = azurerm.billing + alias = "foofrix-azure-devtest-subscription" + subscription_name = "FooFrix Azure DevTest Subscription" + billing_scope_id = local.billing_scope_id + workload = "DevTest" + tags = local.common_tags + + timeouts { + create = "60m" + } +} + +data "azuread_group" "relops" { + display_name = "Relops" +} + +data "azuread_service_principal" "foofrix" { + display_name = "sp-foofrix-azure-devtest" +} + +data "azuread_user" "perf" { + for_each = local.perf_members + user_principal_name = each.value +} + +resource "azurerm_role_assignment" "relops_owner" { + scope = "/subscriptions/${azurerm_subscription.foofrix.subscription_id}" + role_definition_name = "Owner" + principal_id = data.azuread_group.relops.object_id + principal_type = "Group" +} + +resource "azurerm_role_assignment" "foofrix_contributor" { + scope = "/subscriptions/${azurerm_subscription.foofrix.subscription_id}" + role_definition_name = "Contributor" + principal_id = data.azuread_service_principal.foofrix.object_id + principal_type = "ServicePrincipal" + skip_service_principal_aad_check = true +} + +resource "azurerm_role_assignment" "perf_contributor" { + for_each = data.azuread_user.perf + scope = "/subscriptions/${azurerm_subscription.foofrix.subscription_id}" + role_definition_name = "Contributor" + principal_id = each.value.object_id + principal_type = "User" +} + +resource "azurerm_resource_provider_registration" "this" { + for_each = toset([ + "Microsoft.Compute", + "Microsoft.KeyVault", + "Microsoft.ManagedIdentity", + "Microsoft.Network", + "Microsoft.Quota", + "Microsoft.Storage", + ]) + name = each.value +} + +resource "azurerm_resource_group" "foofrix" { + name = "rg-foofrix" + location = local.location + tags = local.common_tags +} diff --git a/terraform/azure_foofrix/outputs.tf b/terraform/azure_foofrix/outputs.tf new file mode 100644 index 00000000..545385fb --- /dev/null +++ b/terraform/azure_foofrix/outputs.tf @@ -0,0 +1,19 @@ +output "subscription_id" { + value = azurerm_subscription.foofrix.subscription_id +} + +output "provisioner_client_id" { + value = data.azuread_service_principal.foofrix.client_id +} + +output "worker_identity_id" { + value = azurerm_user_assigned_identity.worker.id +} + +output "worker_identity_client_id" { + value = azurerm_user_assigned_identity.worker.client_id +} + +output "key_vault_uri" { + value = azurerm_key_vault.foofrix.vault_uri +} diff --git a/terraform/azure_foofrix/providers.tf b/terraform/azure_foofrix/providers.tf new file mode 100644 index 00000000..0df305a1 --- /dev/null +++ b/terraform/azure_foofrix/providers.tf @@ -0,0 +1,33 @@ +terraform { + required_version = ">= 1.10" + required_providers { + azuread = { + source = "hashicorp/azuread" + version = "~> 3" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4" + } + } +} + +# Use an existing subscription to create the new subscription first. +provider "azurerm" { + alias = "billing" + features {} + resource_provider_registrations = "none" + subscription_id = "108d46d5-fe9b-4850-9a7d-8c914aa6c1f0" + tenant_id = local.tenant_id +} + +provider "azurerm" { + features {} + resource_provider_registrations = "none" + subscription_id = azurerm_subscription.foofrix.subscription_id + tenant_id = local.tenant_id +} + +provider "azuread" { + tenant_id = local.tenant_id +} From ae22226c1a24e9376b5ca906f370c91612b2fc0b Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 13:56:33 -0400 Subject: [PATCH 02/10] RELOPS-2548: Remove Terraform access test --- terraform/azure_foofrix/README.md | 1 - terraform/azure_foofrix/access.tftest.hcl | 47 ----------------------- 2 files changed, 48 deletions(-) delete mode 100644 terraform/azure_foofrix/access.tftest.hcl diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md index 17b1de09..8187288d 100644 --- a/terraform/azure_foofrix/README.md +++ b/terraform/azure_foofrix/README.md @@ -104,5 +104,4 @@ the latest autoland decision task. A new tier 1 regression blocks deployment. terraform init -backend=false terraform fmt -check terraform validate -terraform test ``` diff --git a/terraform/azure_foofrix/access.tftest.hcl b/terraform/azure_foofrix/access.tftest.hcl deleted file mode 100644 index 7efa047d..00000000 --- a/terraform/azure_foofrix/access.tftest.hcl +++ /dev/null @@ -1,47 +0,0 @@ -mock_provider "azurerm" { - mock_resource "azurerm_key_vault" { - defaults = { - id = "/subscriptions/11111111-2222-3333-4444-555555555555/resourceGroups/rg-foofrix/providers/Microsoft.KeyVault/vaults/kv-foofrix-test" - } - } -} -mock_provider "azurerm" { - alias = "billing" -} -mock_provider "azuread" {} - -override_resource { - target = azurerm_subscription.foofrix - values = { - subscription_id = "11111111-2222-3333-4444-555555555555" - } -} - -run "access_boundaries" { - # All providers are mocked; this does not create cloud resources. - command = apply - - assert { - condition = ( - azurerm_role_assignment.foofrix_contributor.scope == "/subscriptions/11111111-2222-3333-4444-555555555555" && - azurerm_role_assignment.foofrix_contributor.role_definition_name == "Contributor" - ) - error_message = "The GCP provisioner must have Contributor access only in FooFrix." - } - - assert { - condition = ( - azurerm_role_assignment.worker_secrets_user.scope == azurerm_key_vault.foofrix.id && - azurerm_role_assignment.worker_secrets_user.role_definition_name == "Key Vault Secrets User" - ) - error_message = "The worker must have read access at the FooFrix vault scope." - } - - assert { - condition = ( - azurerm_role_assignment.relops_owner.scope == "/subscriptions/11111111-2222-3333-4444-555555555555" && - azurerm_role_assignment.relops_owner.role_definition_name == "Owner" - ) - error_message = "RelOps must retain ownership of the new subscription." - } -} From 897146fb5e02926611ab67f398de7ed3aa54b205 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 14:00:58 -0400 Subject: [PATCH 03/10] RELOPS-2548: Document existing billing export coverage --- terraform/azure_foofrix/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md index 8187288d..e823a1a0 100644 --- a/terraform/azure_foofrix/README.md +++ b/terraform/azure_foofrix/README.md @@ -5,6 +5,13 @@ This stack implements the subscription plan in the It follows `azure_fuzzing`: `azure_ad` owns the application and service principal; this stack owns the subscription and Azure resources. +FooFrix uses the same billing profile (`GRUW-TLBL-BG7-PGB`) and invoice section +(`VVEC-AWWS-PJA-PGB`) as fuzzing. The existing daily Actual Cost, Amortized Cost, +and FOCUS exports in `azure_billing/finops.tf` cover these scopes without +subscription filters. They write to `safinopsdata/cost-management`. No separate +export is needed. After deployment and billing data arrival, filter by the +FooFrix subscription ID to report its costs. + The draft uses Central US and gives Denis Palmeiro access. Confirm the region and full Perf access list before deployment. Perf will create and remove its VMs. This stack does not create a Taskcluster pool. From a063b0af7d58ee9cea00d6f2500042bb93374ab0 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 14:06:49 -0400 Subject: [PATCH 04/10] RELOPS-2548: Use service principal access for FooFrix provisioning --- terraform/azure_ad/foofrix.tf | 2 +- terraform/azure_foofrix/README.md | 24 ++++++++++++------------ terraform/azure_foofrix/keyvault.tf | 12 ++++++------ terraform/azure_foofrix/main.tf | 16 ---------------- 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/terraform/azure_ad/foofrix.tf b/terraform/azure_ad/foofrix.tf index 06c2ce31..e92138e7 100644 --- a/terraform/azure_ad/foofrix.tf +++ b/terraform/azure_ad/foofrix.tf @@ -1,5 +1,5 @@ # FooFrix identities. Subscription roles are in azure_foofrix. RELOPS-2548. -# Add GCP federation when the service account unique ID is known. +# Create the client secret outside Terraform and store it in 1Password. resource "azuread_application" "foofrix" { display_name = "sp-foofrix-azure-devtest" owners = data.azuread_group.relops.members diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md index e823a1a0..12b5f6d3 100644 --- a/terraform/azure_foofrix/README.md +++ b/terraform/azure_foofrix/README.md @@ -12,19 +12,17 @@ subscription filters. They write to `safinopsdata/cost-management`. No separate export is needed. After deployment and billing data arrival, filter by the FooFrix subscription ID to report its costs. -The draft uses Central US and gives Denis Palmeiro access. Confirm the region -and full Perf access list before deployment. Perf will create and remove its -VMs. This stack does not create a Taskcluster pool. +The draft uses Central US. The team will use the service principal to create +and remove VMs. No new team group or individual access grant is needed. | Identity | Access | | --- | --- | | Relops group | Subscription Owner; Key Vault Administrator | -| Perf members in `main.tf` | Subscription Contributor; Key Vault Secrets Officer | -| `sp-foofrix-azure-devtest` | Contributor on the FooFrix subscription | +| `sp-foofrix-azure-devtest` | Subscription Contributor; Key Vault Secrets Officer | | `id-foofrix-worker` | Read secrets in the FooFrix vault | Attach `id-foofrix-worker` to each FooFrix VM. Use its client ID to select it -when the agent reads Key Vault. Perf members can add AI keys through Key Vault. +when the agent reads Key Vault. The provisioner can add AI keys to the vault. Keep secret values out of Terraform, VM images, and startup scripts. Contributor access lets the provisioner attach this identity without permission to create role assignments. It therefore also permits indirect access to these secrets @@ -37,8 +35,10 @@ Use two stages for the first deployment. The `billing` provider uses the existin FXCI subscription only to call the subscription creation API. 1. In `azure_ad`, review and apply the FooFrix application and service - principal. It has no credential yet. Add GCP federation after the service - account unique ID is known. + principal. Create its client secret outside Terraform and store it in the + RelOps 1Password vault, as for fuzzing. Record its expiry and renewal owner. + Give the team the tenant ID, client ID, subscription ID, and secret through + the approved secret-sharing process. 2. In this directory, initialize the backend and review the subscription plan: ```sh @@ -60,10 +60,10 @@ FXCI subscription only to call the subscription creation API. ## GCP authentication -For GCP to Azure, add a federated credential to the FooFrix application once -Perf supplies the GCP service account's numeric unique ID. Use issuer -`https://accounts.google.com` and audience `api://AzureADTokenExchange`. -See the [Microsoft GCP federation guide](https://learn.microsoft.com/entra/workload-id/workload-identity-federation-google-cloud). +The GCP provisioner can authenticate with the FooFrix tenant ID, application +client ID, and client secret. Store the secret in its secret store. No GCP +service account ID is required for this Azure login. GCP federation can be +added later if needed. For Azure to GCS, use the worker managed identity with Google Workload Identity Federation. This still needs the GCP project, results bucket, and required object diff --git a/terraform/azure_foofrix/keyvault.tf b/terraform/azure_foofrix/keyvault.tf index 70eb16cc..c82658ad 100644 --- a/terraform/azure_foofrix/keyvault.tf +++ b/terraform/azure_foofrix/keyvault.tf @@ -27,12 +27,12 @@ resource "azurerm_role_assignment" "relops_key_vault_administrator" { principal_type = "Group" } -resource "azurerm_role_assignment" "perf_secrets_officer" { - for_each = data.azuread_user.perf - scope = azurerm_key_vault.foofrix.id - role_definition_name = "Key Vault Secrets Officer" - principal_id = each.value.object_id - principal_type = "User" +resource "azurerm_role_assignment" "foofrix_secrets_officer" { + scope = azurerm_key_vault.foofrix.id + role_definition_name = "Key Vault Secrets Officer" + principal_id = data.azuread_service_principal.foofrix.object_id + principal_type = "ServicePrincipal" + skip_service_principal_aad_check = true } resource "azurerm_role_assignment" "worker_secrets_user" { diff --git a/terraform/azure_foofrix/main.tf b/terraform/azure_foofrix/main.tf index bbb01826..5a544c7c 100644 --- a/terraform/azure_foofrix/main.tf +++ b/terraform/azure_foofrix/main.tf @@ -7,9 +7,6 @@ locals { mozilla_invoice_section_id = "VVEC-AWWS-PJA-PGB" billing_scope_id = "/providers/Microsoft.Billing/billingAccounts/${local.billing_account_id}/billingProfiles/${local.mozilla_billing_profile_id}/invoiceSections/${local.mozilla_invoice_section_id}" - # Confirm the complete Perf access list before deployment. - perf_members = toset(["dpalmeiro@mozilla.com"]) - common_tags = { terraform = "true" project_name = "azure_foofrix" @@ -40,11 +37,6 @@ data "azuread_service_principal" "foofrix" { display_name = "sp-foofrix-azure-devtest" } -data "azuread_user" "perf" { - for_each = local.perf_members - user_principal_name = each.value -} - resource "azurerm_role_assignment" "relops_owner" { scope = "/subscriptions/${azurerm_subscription.foofrix.subscription_id}" role_definition_name = "Owner" @@ -60,14 +52,6 @@ resource "azurerm_role_assignment" "foofrix_contributor" { skip_service_principal_aad_check = true } -resource "azurerm_role_assignment" "perf_contributor" { - for_each = data.azuread_user.perf - scope = "/subscriptions/${azurerm_subscription.foofrix.subscription_id}" - role_definition_name = "Contributor" - principal_id = each.value.object_id - principal_type = "User" -} - resource "azurerm_resource_provider_registration" "this" { for_each = toset([ "Microsoft.Compute", From fc58ca6eb8bda3280d20f169b91775aae14f1a20 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 14:11:52 -0400 Subject: [PATCH 05/10] RELOPS-2548: Explain FooFrix subscription purpose in README --- terraform/azure_foofrix/README.md | 122 +++++------------------------- 1 file changed, 19 insertions(+), 103 deletions(-) diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md index 12b5f6d3..2bcae9f0 100644 --- a/terraform/azure_foofrix/README.md +++ b/terraform/azure_foofrix/README.md @@ -1,114 +1,30 @@ # FooFrix Azure subscription -This stack implements the subscription plan in the -[September 11 meeting notes](https://mozilla-hub.atlassian.net/browse/RELOPS-2548?focusedCommentId=1727342). -It follows `azure_fuzzing`: `azure_ad` owns the application and service principal; -this stack owns the subscription and Azure resources. +FooFrix runs agents that profile Firefox, test performance changes, build +Firefox, and produce patches. This dedicated Azure DevTest subscription lets +the team create and manage Windows VMs for runs that can last more than 24 hours. +It gives the team control over VM sizes and lifetimes, with separate costs and +access for FooFrix. -FooFrix uses the same billing profile (`GRUW-TLBL-BG7-PGB`) and invoice section -(`VVEC-AWWS-PJA-PGB`) as fuzzing. The existing daily Actual Cost, Amortized Cost, -and FOCUS exports in `azure_billing/finops.tf` cover these scopes without -subscription filters. They write to `safinopsdata/cost-management`. No separate -export is needed. After deployment and billing data arrival, filter by the -FooFrix subscription ID to report its costs. - -The draft uses Central US. The team will use the service principal to create -and remove VMs. No new team group or individual access grant is needed. +This Terraform stack manages the subscription, a resource group in Central US, +a Key Vault for AI keys and other secrets, and a managed identity for the VMs. +The team manages the VMs through `sp-foofrix-azure-devtest`. The application and +service principal are managed in `../azure_ad/foofrix.tf`. | Identity | Access | | --- | --- | -| Relops group | Subscription Owner; Key Vault Administrator | +| Existing Relops group | Subscription Owner; Key Vault Administrator | | `sp-foofrix-azure-devtest` | Subscription Contributor; Key Vault Secrets Officer | | `id-foofrix-worker` | Read secrets in the FooFrix vault | -Attach `id-foofrix-worker` to each FooFrix VM. Use its client ID to select it -when the agent reads Key Vault. The provisioner can add AI keys to the vault. -Keep secret values out of Terraform, VM images, and startup scripts. Contributor -access lets the provisioner attach this identity without permission to create -role assignments. It therefore also permits indirect access to these secrets -through a VM that it controls. - -## Initial deployment - -The subscription must exist before the default Azure provider can use it. -Use two stages for the first deployment. The `billing` provider uses the existing -FXCI subscription only to call the subscription creation API. - -1. In `azure_ad`, review and apply the FooFrix application and service - principal. Create its client secret outside Terraform and store it in the - RelOps 1Password vault, as for fuzzing. Record its expiry and renewal owner. - Give the team the tenant ID, client ID, subscription ID, and secret through - the approved secret-sharing process. -2. In this directory, initialize the backend and review the subscription plan: - - ```sh - export AWS_PROFILE=AdministratorAccess-961225894672 - terraform init - terraform plan -target=azurerm_subscription.foofrix -out=subscription.tfplan - ``` - -3. Apply the reviewed subscription plan. Then review a full plan: - - ```sh - terraform apply subscription.tfplan - terraform plan -out=foofrix.tfplan - ``` - -4. Apply the reviewed full plan. Use full plans for later changes. - Give Perf the subscription ID, provisioner client ID, worker identity ID, - worker identity client ID, and vault URI from `terraform output`. - -## GCP authentication - -The GCP provisioner can authenticate with the FooFrix tenant ID, application -client ID, and client secret. Store the secret in its secret store. No GCP -service account ID is required for this Azure login. GCP federation can be -added later if needed. - -For Azure to GCS, use the worker managed identity with Google Workload Identity -Federation. This still needs the GCP project, results bucket, and required object -operations. Configure the Entra audience application, Google trust provider, -identity restriction, and bucket access after those values are known. These -resources are not part of this draft. See the -[Google Azure federation guide](https://docs.cloud.google.com/iam/docs/workload-identity-federation-with-other-clouds). - -## Windows VM and image work - -Start with one regular VM. Do not configure Spot eviction or a one-hour shutdown. -The first run must last at least 24 hours and complete a Firefox build and a -FooFrix test. Perf can then increase the count to two or three. - -For a GPU proof of concept, evaluate `Standard_NV18ads_A10_v5` (18 vCPUs, -220 GiB RAM, half an A10 GPU). If the test needs a full GPU, evaluate -`Standard_NV36ads_A10_v5` (36 vCPUs, 440 GiB RAM, one A10). Confirm regional -availability and quota in the new subscription. These are candidates, pending -the harness requirements. See the [Azure size table](https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/gpu-accelerated/nvadsa10v5-series). -Start with a 1 TiB persistent build disk and measure peak use. Temporary storage -must not hold the only copy of source changes or results. - -Provisioning needs Azure CLI or an Azure SDK, a VNet and subnet, a restricted -remote access rule, a Windows image version, persistent disks, and the worker -identity. Select the exact image and access method after repository inspection. - -`worker-images` has the Packer and Azure Compute Gallery build path. Its Windows -configs select Puppet roles and Pester tests. Its GitHub workflows also check -`.github/relsre.json`; repository access alone does not permit a build. - -Before adding a FooFrix image, inspect the harness to determine whether it needs -a prebuilt Chromium release or a Chromium source build with release options. -Confirm the version, build flags, toolchain, GPU driver, expected paths, and -update process. Add a separate image config and checks for Firefox builds, -Chromium startup, GPU use, and long VM lifetime. Confirm that Taskcluster startup -and shutdown services cannot terminate the standalone VM. Arrange gallery read -access from the FooFrix subscription and build access for the named Perf users. - -Any later use in a production Firefox CI pool must pass all tier 1 tasks from -the latest autoland decision task. A new tier 1 regression blocks deployment. +The provisioning service uses a tenant ID, client ID, and client secret to +access Azure. The client secret is managed outside Terraform and stored in +1Password. VMs use `id-foofrix-worker` to read secrets from Key Vault. -## Checks +FooFrix uses the same Mozilla billing profile and invoice section as fuzzing. +The daily Actual Cost, Amortized Cost, and FOCUS exports in +`../azure_billing/finops.tf` include its costs in +`safinopsdata/cost-management`. Filter by the FooFrix subscription ID to report +its costs. -```sh -terraform init -backend=false -terraform fmt -check -terraform validate -``` +Related issue: [RELOPS-2548](https://mozilla-hub.atlassian.net/browse/RELOPS-2548). From 4c1f0f1b02b564827ecb9d539a6f9a087a19caad Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 14:12:15 -0400 Subject: [PATCH 06/10] RELOPS-2548: Link to the FooFrix tool --- terraform/azure_foofrix/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md index 2bcae9f0..c1477898 100644 --- a/terraform/azure_foofrix/README.md +++ b/terraform/azure_foofrix/README.md @@ -1,6 +1,7 @@ # FooFrix Azure subscription -FooFrix runs agents that profile Firefox, test performance changes, build +[FooFrix](https://foofrix.uc.r.appspot.com/?component=JS&suite=speedometer3) +runs agents that profile Firefox, test performance changes, build Firefox, and produce patches. This dedicated Azure DevTest subscription lets the team create and manage Windows VMs for runs that can last more than 24 hours. It gives the team control over VM sizes and lifetimes, with separate costs and From 34f7928bcc31939276e44c639eaee47ccc3d911d Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 14:18:13 -0400 Subject: [PATCH 07/10] RELOPS-2548: Add FooFrix image gallery and artifact storage --- terraform/azure_foofrix/README.md | 11 +++++-- terraform/azure_foofrix/images.tf | 52 ++++++++++++++++++++++++++++++ terraform/azure_foofrix/outputs.tf | 8 +++++ 3 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 terraform/azure_foofrix/images.tf diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md index c1477898..7c7cf539 100644 --- a/terraform/azure_foofrix/README.md +++ b/terraform/azure_foofrix/README.md @@ -9,18 +9,23 @@ access for FooFrix. This Terraform stack manages the subscription, a resource group in Central US, a Key Vault for AI keys and other secrets, and a managed identity for the VMs. +The `foofrix` Compute Gallery stores VM image versions. A private `artifacts` +Blob Storage container holds build files and test results in Standard LRS storage. The team manages the VMs through `sp-foofrix-azure-devtest`. The application and service principal are managed in `../azure_ad/foofrix.tf`. | Identity | Access | | --- | --- | -| Existing Relops group | Subscription Owner; Key Vault Administrator | -| `sp-foofrix-azure-devtest` | Subscription Contributor; Key Vault Secrets Officer | -| `id-foofrix-worker` | Read secrets in the FooFrix vault | +| Existing Relops group | Subscription Owner; Key Vault Administrator; blob read/write | +| `sp-foofrix-azure-devtest` | Subscription Contributor; Key Vault Secrets Officer; blob read/write | +| `id-foofrix-worker` | Read vault secrets; blob read/write | The provisioning service uses a tenant ID, client ID, and client secret to access Azure. The client secret is managed outside Terraform and stored in 1Password. VMs use `id-foofrix-worker` to read secrets from Key Vault. +Blob access uses these identities through the Storage Blob Data Contributor +role on the `artifacts` container. The provisioning service can manage gallery +images through its subscription Contributor role. FooFrix uses the same Mozilla billing profile and invoice section as fuzzing. The daily Actual Cost, Amortized Cost, and FOCUS exports in diff --git a/terraform/azure_foofrix/images.tf b/terraform/azure_foofrix/images.tf new file mode 100644 index 00000000..4dac991b --- /dev/null +++ b/terraform/azure_foofrix/images.tf @@ -0,0 +1,52 @@ +resource "azurerm_shared_image_gallery" "foofrix" { + name = "foofrix" + resource_group_name = azurerm_resource_group.foofrix.name + location = local.location + description = "Windows images for FooFrix performance agents." + tags = local.common_tags + + depends_on = [azurerm_resource_provider_registration.this["Microsoft.Compute"]] +} + +resource "azurerm_storage_account" "foofrix" { + name = "safoofrix${substr(azurerm_subscription.foofrix.subscription_id, 0, 8)}" + resource_group_name = azurerm_resource_group.foofrix.name + location = local.location + account_tier = "Standard" + account_replication_type = "LRS" + account_kind = "StorageV2" + min_tls_version = "TLS1_2" + allow_nested_items_to_be_public = false + tags = local.common_tags + + depends_on = [azurerm_resource_provider_registration.this["Microsoft.Storage"]] +} + +resource "azurerm_storage_container" "artifacts" { + name = "artifacts" + storage_account_id = azurerm_storage_account.foofrix.id + container_access_type = "private" +} + +resource "azurerm_role_assignment" "blob_contributor" { + for_each = { + provisioner = { + id = data.azuread_service_principal.foofrix.object_id + type = "ServicePrincipal" + } + worker = { + id = azurerm_user_assigned_identity.worker.principal_id + type = "ServicePrincipal" + } + relops = { + id = data.azuread_group.relops.object_id + type = "Group" + } + } + + scope = azurerm_storage_container.artifacts.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = each.value.id + principal_type = each.value.type + skip_service_principal_aad_check = each.value.type == "ServicePrincipal" +} diff --git a/terraform/azure_foofrix/outputs.tf b/terraform/azure_foofrix/outputs.tf index 545385fb..aca85b68 100644 --- a/terraform/azure_foofrix/outputs.tf +++ b/terraform/azure_foofrix/outputs.tf @@ -17,3 +17,11 @@ output "worker_identity_client_id" { output "key_vault_uri" { value = azurerm_key_vault.foofrix.vault_uri } + +output "image_gallery_id" { + value = azurerm_shared_image_gallery.foofrix.id +} + +output "artifacts_container_url" { + value = "${azurerm_storage_account.foofrix.primary_blob_endpoint}${azurerm_storage_container.artifacts.name}" +} From fcf751e7e9c88c45b66e644d9f8bbf2253aefb79 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 14:33:30 -0400 Subject: [PATCH 08/10] RELOPS-2548: Link FooFrix source repository --- terraform/azure_foofrix/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md index 7c7cf539..9f53b726 100644 --- a/terraform/azure_foofrix/README.md +++ b/terraform/azure_foofrix/README.md @@ -7,6 +7,9 @@ the team create and manage Windows VMs for runs that can last more than 24 hours It gives the team control over VM sizes and lifetimes, with separate costs and access for FooFrix. +The harness source and image provisioning scripts are in +[dpalmeiro/foofrix](https://github.com/dpalmeiro/foofrix). + This Terraform stack manages the subscription, a resource group in Central US, a Key Vault for AI keys and other secrets, and a managed identity for the VMs. The `foofrix` Compute Gallery stores VM image versions. A private `artifacts` From abcc4f334d6151c1091a4ba370cac7753605a2af Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 14:39:11 -0400 Subject: [PATCH 09/10] RELOPS-2548: Clarify Azure as a Windows target for GCP FooFrix --- terraform/azure_foofrix/README.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md index 9f53b726..38043add 100644 --- a/terraform/azure_foofrix/README.md +++ b/terraform/azure_foofrix/README.md @@ -1,20 +1,22 @@ # FooFrix Azure subscription [FooFrix](https://foofrix.uc.r.appspot.com/?component=JS&suite=speedometer3) -runs agents that profile Firefox, test performance changes, build -Firefox, and produce patches. This dedicated Azure DevTest subscription lets -the team create and manage Windows VMs for runs that can last more than 24 hours. -It gives the team control over VM sizes and lifetimes, with separate costs and -access for FooFrix. +runs agents that profile Firefox, test performance changes, build Firefox, +and produce patches. Its launcher, scheduler, queue, results, and dashboard +remain in GCP. This dedicated Azure DevTest subscription gives the GCP launcher +a target for Windows testing. It can create VMs from an image, run tests, and +remove the VMs when finished, including runs that last more than 24 hours. +Windows workers use the existing GCS queue and report results to +`gs://foofrix-findings`. The harness source and image provisioning scripts are in [dpalmeiro/foofrix](https://github.com/dpalmeiro/foofrix). This Terraform stack manages the subscription, a resource group in Central US, -a Key Vault for AI keys and other secrets, and a managed identity for the VMs. +a Key Vault available for Windows worker secrets, and a managed identity for the VMs. The `foofrix` Compute Gallery stores VM image versions. A private `artifacts` -Blob Storage container holds build files and test results in Standard LRS storage. -The team manages the VMs through `sp-foofrix-azure-devtest`. The application and +Blob Storage container holds Azure build and image files in Standard LRS storage. +The GCP launcher manages the VMs through `sp-foofrix-azure-devtest`. The application and service principal are managed in `../azure_ad/foofrix.tf`. | Identity | Access | @@ -23,9 +25,10 @@ service principal are managed in `../azure_ad/foofrix.tf`. | `sp-foofrix-azure-devtest` | Subscription Contributor; Key Vault Secrets Officer; blob read/write | | `id-foofrix-worker` | Read vault secrets; blob read/write | -The provisioning service uses a tenant ID, client ID, and client secret to +The GCP launcher uses a tenant ID, client ID, and client secret to access Azure. The client secret is managed outside Terraform and stored in -1Password. VMs use `id-foofrix-worker` to read secrets from Key Vault. +1Password. VMs can use `id-foofrix-worker` to read secrets from Key Vault. +Access to the GCS queue and results bucket requires separate Google credentials. Blob access uses these identities through the Storage Blob Data Contributor role on the `artifacts` container. The provisioning service can manage gallery images through its subscription Contributor role. From 6688146eca1a31641ac6cf5077109c5252b22d92 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Mon, 14 Sep 2026 15:28:42 -0400 Subject: [PATCH 10/10] RELOPS-2548 Add FooFrix team access and image build resources --- terraform/azure_ad/foofrix.tf | 36 +++++++++ terraform/azure_foofrix/README.md | 47 +++++++++++- terraform/azure_foofrix/images.tf | 73 +++++++++++++++++++ terraform/azure_foofrix/keyvault.tf | 7 ++ terraform/azure_foofrix/main.tf | 12 +++ terraform/azure_foofrix/outputs.tf | 28 +++++++ .../azure_foofrix/tests/access.tftest.hcl | 59 +++++++++++++++ 7 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 terraform/azure_foofrix/tests/access.tftest.hcl diff --git a/terraform/azure_ad/foofrix.tf b/terraform/azure_ad/foofrix.tf index e92138e7..fa117f42 100644 --- a/terraform/azure_ad/foofrix.tf +++ b/terraform/azure_ad/foofrix.tf @@ -11,3 +11,39 @@ resource "azuread_service_principal" "foofrix" { owners = data.azuread_group.relops.members tags = concat(["name:sp-foofrix-azure-devtest"], local.sp_tags) } + +resource "azuread_group" "platform_performance" { + display_name = "Platform Performance" + security_enabled = true + mail_enabled = false + description = "Managed by RelOps - Platform Performance team" +} + +resource "azuread_group_member" "platform_performance" { + for_each = { + dpalmeiro = "2e8c6f6d-9dae-42b3-a193-5ea7c4b09cb5" + jlink = "d76c0d0a-537a-42a3-9ac7-0d96caa8e054" + } + group_object_id = azuread_group.platform_performance.object_id + member_object_id = each.value +} + +resource "azuread_application" "foofrix_image_build" { + display_name = "sp-foofrix-image-build" + owners = data.azuread_group.relops.members + notes = "FooFrix Windows image builds from worker-images. RELOPS-2570." +} + +resource "azuread_service_principal" "foofrix_image_build" { + client_id = azuread_application.foofrix_image_build.client_id + owners = data.azuread_group.relops.members + tags = concat(["name:sp-foofrix-image-build"], local.sp_tags) +} + +resource "azuread_application_federated_identity_credential" "foofrix_image_build" { + application_id = azuread_application.foofrix_image_build.id + display_name = "github-worker-images-foofrix" + audiences = ["api://AzureADTokenExchange"] + issuer = "https://token.actions.githubusercontent.com" + subject = "repo:mozilla-platform-ops/worker-images:environment:foofrix-image-build" +} diff --git a/terraform/azure_foofrix/README.md b/terraform/azure_foofrix/README.md index 38043add..6d033dbb 100644 --- a/terraform/azure_foofrix/README.md +++ b/terraform/azure_foofrix/README.md @@ -14,7 +14,10 @@ The harness source and image provisioning scripts are in This Terraform stack manages the subscription, a resource group in Central US, a Key Vault available for Windows worker secrets, and a managed identity for the VMs. -The `foofrix` Compute Gallery stores VM image versions. A private `artifacts` +Terraform manages the `foofrix` Compute Gallery and its `win11_64_24h2` image +definition. It uses the existing FXCI Windows 11 24H2 properties: Windows, x64, +Hyper-V V2, generalized, and `MicrosoftWindowsDesktop/Windows-11/win11-24h2-avd`. +The worker-images workflow publishes image versions. A private `artifacts` Blob Storage container holds Azure build and image files in Standard LRS storage. The GCP launcher manages the VMs through `sp-foofrix-azure-devtest`. The application and service principal are managed in `../azure_ad/foofrix.tf`. @@ -24,6 +27,9 @@ service principal are managed in `../azure_ad/foofrix.tf`. | Existing Relops group | Subscription Owner; Key Vault Administrator; blob read/write | | `sp-foofrix-azure-devtest` | Subscription Contributor; Key Vault Secrets Officer; blob read/write | | `id-foofrix-worker` | Read vault secrets; blob read/write | +| Platform Performance | Subscription Contributor; Key Vault Secrets Officer; blob read/write | +| `sp-foofrix-image-build` | Contributor on the build resource group and gallery; blob read; attach the build identity | +| `id-foofrix-image-build` | Blob read during image creation | The GCP launcher uses a tenant ID, client ID, and client secret to access Azure. The client secret is managed outside Terraform and stored in @@ -40,3 +46,42 @@ The daily Actual Cost, Amortized Cost, and FOCUS exports in its costs. Related issue: [RELOPS-2548](https://mozilla-hub.atlassian.net/browse/RELOPS-2548). + +## Image builds and access setup + +Apply `azure_ad` before this stack. Create the subscription with a targeted plan +for `azurerm_subscription.foofrix`, then run a full plan and apply. + +The build application uses GitHub OIDC with this exact subject: +`repo:mozilla-platform-ops/worker-images:environment:foofrix-image-build`. +RELOPS-2570 must create and protect that dedicated GitHub environment and use it +in the authorized FooFrix workflow before image builds start. This subject +permits jobs that use that environment; it does not identify a workflow file. +Use environment deployment rules and the team authorization check to control access. +The workflow needs `id-token: write` and audience `api://AzureADTokenExchange`. +No image-build client secret is needed. + +Configure Packer to use the existing `image_build_resource_group` output for +temporary resources. Publish to `image_gallery_name` in +`image_gallery_resource_group`, using the definition from +`windows_image_definition_id`. The workflow logs in with `image_build_client_id`. +Attach `image_build_identity_id` to the temporary VM. The guest bootstrap must +use that managed identity to authenticate artifact downloads, with +`image_build_identity_client_id` to select it. The GitHub login does not provide +credentials inside the VM. Both build identities have read access to `artifacts`. + +Platform Performance starts with Denis Palmeiro and Justin Link. Add Frank Doty, +Andrew Creskey, Jamie Nicol, Marc Leclair, Markus Stange, and Sky Ning after their +Entra accounts are created. Deliver Denis's temporary password through the +approved private process. He must change it and complete MFA enrollment. +Do not put credentials in Terraform or the PR. + +Perf owns VM creation, deletion, and the FooFrix harness and GCP integration. +RelOps supplies the Azure resources and image-build path. RELOPS-2570 covers +Windows tooling, profiling support, startup, and image validation. + +The GCS authentication design still needs agreement with Perf. If Azure managed +identity to Google Workload Identity Federation is selected, add the Entra +audience application in `azure_ad` and the Google trust and bucket grants in GCP. +Queue and state access to `foofrix-findings` in project `foofrix` needs reads, +updates, and deletes as well as uploads. Test results remain in GCS. diff --git a/terraform/azure_foofrix/images.tf b/terraform/azure_foofrix/images.tf index 4dac991b..d1daa5ab 100644 --- a/terraform/azure_foofrix/images.tf +++ b/terraform/azure_foofrix/images.tf @@ -38,6 +38,10 @@ resource "azurerm_role_assignment" "blob_contributor" { id = azurerm_user_assigned_identity.worker.principal_id type = "ServicePrincipal" } + platform_performance = { + id = data.azuread_group.platform_performance.object_id + type = "Group" + } relops = { id = data.azuread_group.relops.object_id type = "Group" @@ -50,3 +54,72 @@ resource "azurerm_role_assignment" "blob_contributor" { principal_type = each.value.type skip_service_principal_aad_check = each.value.type == "ServicePrincipal" } + +resource "azurerm_shared_image" "windows" { + name = "win11_64_24h2" + gallery_name = azurerm_shared_image_gallery.foofrix.name + resource_group_name = azurerm_resource_group.foofrix.name + location = local.location + os_type = "Windows" + architecture = "x64" + hyper_v_generation = "V2" + specialized = false + tags = local.common_tags + + identifier { + publisher = "MicrosoftWindowsDesktop" + offer = "Windows-11" + sku = "win11-24h2-avd" + } +} + +data "azuread_service_principal" "foofrix_image_build" { + display_name = "sp-foofrix-image-build" +} + +resource "azurerm_resource_group" "image_build" { + name = "rg-foofrix-image-build" + location = local.location + tags = local.common_tags +} + +resource "azurerm_user_assigned_identity" "image_build" { + name = "id-foofrix-image-build" + resource_group_name = azurerm_resource_group.foofrix.name + location = local.location + tags = local.common_tags + + depends_on = [azurerm_resource_provider_registration.this["Microsoft.ManagedIdentity"]] +} + +resource "azurerm_role_assignment" "image_build_contributor" { + for_each = { + build = azurerm_resource_group.image_build.id + gallery = azurerm_shared_image_gallery.foofrix.id + } + scope = each.value + role_definition_name = "Contributor" + principal_id = data.azuread_service_principal.foofrix_image_build.object_id + principal_type = "ServicePrincipal" + skip_service_principal_aad_check = true +} + +resource "azurerm_role_assignment" "image_build_identity_operator" { + scope = azurerm_user_assigned_identity.image_build.id + role_definition_name = "Managed Identity Operator" + principal_id = data.azuread_service_principal.foofrix_image_build.object_id + principal_type = "ServicePrincipal" + skip_service_principal_aad_check = true +} + +resource "azurerm_role_assignment" "image_build_blob_reader" { + for_each = { + builder = data.azuread_service_principal.foofrix_image_build.object_id + guest = azurerm_user_assigned_identity.image_build.principal_id + } + scope = azurerm_storage_container.artifacts.id + role_definition_name = "Storage Blob Data Reader" + principal_id = each.value + principal_type = "ServicePrincipal" + skip_service_principal_aad_check = true +} diff --git a/terraform/azure_foofrix/keyvault.tf b/terraform/azure_foofrix/keyvault.tf index c82658ad..9ef553b9 100644 --- a/terraform/azure_foofrix/keyvault.tf +++ b/terraform/azure_foofrix/keyvault.tf @@ -42,3 +42,10 @@ resource "azurerm_role_assignment" "worker_secrets_user" { principal_type = "ServicePrincipal" skip_service_principal_aad_check = true } + +resource "azurerm_role_assignment" "platform_performance_secrets_officer" { + scope = azurerm_key_vault.foofrix.id + role_definition_name = "Key Vault Secrets Officer" + principal_id = data.azuread_group.platform_performance.object_id + principal_type = "Group" +} diff --git a/terraform/azure_foofrix/main.tf b/terraform/azure_foofrix/main.tf index 5a544c7c..310d4f25 100644 --- a/terraform/azure_foofrix/main.tf +++ b/terraform/azure_foofrix/main.tf @@ -69,3 +69,15 @@ resource "azurerm_resource_group" "foofrix" { location = local.location tags = local.common_tags } + +data "azuread_group" "platform_performance" { + display_name = "Platform Performance" + security_enabled = true +} + +resource "azurerm_role_assignment" "platform_performance_contributor" { + scope = "/subscriptions/${azurerm_subscription.foofrix.subscription_id}" + role_definition_name = "Contributor" + principal_id = data.azuread_group.platform_performance.object_id + principal_type = "Group" +} diff --git a/terraform/azure_foofrix/outputs.tf b/terraform/azure_foofrix/outputs.tf index aca85b68..7d1ef253 100644 --- a/terraform/azure_foofrix/outputs.tf +++ b/terraform/azure_foofrix/outputs.tf @@ -25,3 +25,31 @@ output "image_gallery_id" { output "artifacts_container_url" { value = "${azurerm_storage_account.foofrix.primary_blob_endpoint}${azurerm_storage_container.artifacts.name}" } + +output "image_gallery_name" { + value = azurerm_shared_image_gallery.foofrix.name +} + +output "image_gallery_resource_group" { + value = azurerm_resource_group.foofrix.name +} + +output "windows_image_definition_id" { + value = azurerm_shared_image.windows.id +} + +output "image_build_client_id" { + value = data.azuread_service_principal.foofrix_image_build.client_id +} + +output "image_build_resource_group" { + value = azurerm_resource_group.image_build.name +} + +output "image_build_identity_id" { + value = azurerm_user_assigned_identity.image_build.id +} + +output "image_build_identity_client_id" { + value = azurerm_user_assigned_identity.image_build.client_id +} diff --git a/terraform/azure_foofrix/tests/access.tftest.hcl b/terraform/azure_foofrix/tests/access.tftest.hcl new file mode 100644 index 00000000..03bba726 --- /dev/null +++ b/terraform/azure_foofrix/tests/access.tftest.hcl @@ -0,0 +1,59 @@ +# Run with: terraform -chdir=terraform/azure_foofrix test +# Mock providers keep this access check offline. +mock_provider "azuread" {} +mock_provider "azurerm" {} +mock_provider "azurerm" { + alias = "billing" +} + +override_resource { + target = azurerm_resource_group.image_build + override_during = plan + values = { id = "/subscriptions/00000000-0000-0000-0000-000000000001/resourceGroups/rg-foofrix-image-build" } +} + +override_resource { + target = azurerm_shared_image_gallery.foofrix + override_during = plan + values = { id = "/subscriptions/00000000-0000-0000-0000-000000000001/resourceGroups/rg-foofrix/providers/Microsoft.Compute/galleries/foofrix" } +} + +override_resource { + target = azurerm_user_assigned_identity.image_build + override_during = plan + values = { id = "/subscriptions/00000000-0000-0000-0000-000000000001/resourceGroups/rg-foofrix/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-foofrix-image-build" } +} + +override_resource { + target = azurerm_storage_container.artifacts + override_during = plan + values = { id = "/subscriptions/00000000-0000-0000-0000-000000000001/resourceGroups/rg-foofrix/providers/Microsoft.Storage/storageAccounts/safoofrix/blobServices/default/containers/artifacts" } +} + +run "build_and_team_access" { + command = plan + + assert { + condition = ( + length(azurerm_role_assignment.image_build_contributor) == 2 && + azurerm_role_assignment.image_build_contributor["build"].scope == azurerm_resource_group.image_build.id && + azurerm_role_assignment.image_build_contributor["gallery"].scope == azurerm_shared_image_gallery.foofrix.id && + azurerm_role_assignment.image_build_identity_operator.scope == azurerm_user_assigned_identity.image_build.id && + alltrue([for grant in azurerm_role_assignment.image_build_blob_reader : grant.scope == azurerm_storage_container.artifacts.id]) && + alltrue([for grant in azurerm_role_assignment.image_build_contributor : grant.role_definition_name == "Contributor"]) && + alltrue([for grant in azurerm_role_assignment.image_build_blob_reader : grant.role_definition_name == "Storage Blob Data Reader"]) && + length(azurerm_role_assignment.image_build_blob_reader) == 2 && + azurerm_role_assignment.image_build_identity_operator.role_definition_name == "Managed Identity Operator" + ) + error_message = "The builder needs two Contributor grants, identity attachment, and read access for both build identities." + } + + assert { + condition = ( + azurerm_role_assignment.platform_performance_contributor.role_definition_name == "Contributor" && + azurerm_role_assignment.blob_contributor["platform_performance"].role_definition_name == "Storage Blob Data Contributor" && + azurerm_role_assignment.platform_performance_secrets_officer.role_definition_name == "Key Vault Secrets Officer" + ) + error_message = "Platform Performance needs subscription, artifact, and secret access." + } +}