ci: acn build cil log collector (#4211)

* add cilium log collector to acn official build

* address comments

* fix where we pull cil log collector in acn signed pipeline

signed binaries pipeline pushes to signed images path in our registry and the path
is stored in an environment variable which we need to read and set as the url
previously make cilium-log-collector-version for example wouldn't work because this
pipeline pushes to a different url like
acnpublic.azurecr.io/artifact/XXXXXX/buddy/cilium-log-collector:XXXXXX
instead of
acnpublic.azurecr.io/cilium-log-collector:XXXXXX

Author: QxBytes

.pipelines/build/dockerfiles/cilium-log-collector.Dockerfile

@@ -0,0 +1,5 @@
+ARG ARCH
+
+FROM mcr.microsoft.com/oss/v2/fluent/fluent-bit:v4.2.2 as linux
+ARG ARTIFACT_DIR
+COPY ${ARTIFACT_DIR}/bin/out_azure_app_insights.so /out_azure_app_insights.so

.pipelines/build/ob-prepare.steps.yaml

@@ -66,6 +66,10 @@ steps:
     echo "##vso[task.setvariable variable=azureIptablesMonitorVersion;isOutput=true]$AZUREIPTABLESMONITORVERSION"
     echo "azureIptablesMonitorVersion: $AZUREIPTABLESMONITORVERSION"
 
+    CILIUMLOGCOLLECTORVERSION=$(make cilium-log-collector-version)
+    echo "##vso[task.setvariable variable=ciliumLogCollectorVersion;isOutput=true]$CILIUMLOGCOLLECTORVERSION"
+    echo "ciliumLogCollectorVersion: $CILIUMLOGCOLLECTORVERSION"
+
     AZUREBLOCKIPTABLESVERSION=$(make azure-block-iptables-version)
     echo "##vso[task.setvariable variable=azureBlockIptablesVersion;isOutput=true]$AZUREBLOCKIPTABLESVERSION"
     echo "azureBlockIptablesVersion: $AZUREBLOCKIPTABLESVERSION"

.pipelines/build/scripts/cilium-log-collector.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -eux
+
+[[ $OS =~ windows ]] && { echo "cilium-log-collector is not supported on Windows"; exit 1; }
+# enable cgo for -buildmode=c-shared
+export CGO_ENABLED=1
+
+mkdir -p "$OUT_DIR"/bin
+mkdir -p "$OUT_DIR"/files
+
+echo "Building cilium-log-collector version: $CILIUM_LOG_COLLECTOR_VERSION"
+
+pushd "$REPO_ROOT"/cilium-log-collector
+  GOOS="$OS" go build -buildmode=c-shared -v -a -trimpath \
+    -o "$OUT_DIR"/bin/out_azure_app_insights.so \
+    -ldflags "-s -w -X main.version=$CILIUM_LOG_COLLECTOR_VERSION" \
+    -gcflags="-dwarflocationlists=true" \
+    .
+popd

.pipelines/run-pipeline.yaml

@@ -40,6 +40,7 @@ stages:
     AZURE_IP_MASQ_MERGER_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureIpMasqMergerVersion'] ]
     AZURE_IPTABLES_MONITOR_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureIptablesMonitorVersion'] ]
     AZURE_BLOCK_IPTABLES_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureBlockIptablesVersion'] ]
+    CILIUM_LOG_COLLECTOR_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.ciliumLogCollectorVersion'] ]
     CNI_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.cniVersion'] ]
     CNS_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.cnsVersion'] ]
     IPV6_HP_BPF_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.ipv6HpBpfVersion'] ]
@@ -76,6 +77,12 @@ stages:
               archiveName: azure-iptables-monitor
               archiveVersion: $(AZURE_IPTABLES_MONITOR_VERSION)
               imageTag: $(Build.BuildNumber)
+            cilium_log_collector:
+              name: cilium-log-collector
+              extraArgs: ''
+              archiveName: cilium-log-collector
+              archiveVersion: $(CILIUM_LOG_COLLECTOR_VERSION)
+              imageTag: $(Build.BuildNumber)
             cni:
               name: cni
               extraArgs: '--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID)'
@@ -166,6 +173,12 @@ stages:
               archiveName: azure-iptables-monitor
               archiveVersion: $(AZURE_IPTABLES_MONITOR_VERSION)
               imageTag: $(Build.BuildNumber)
+            cilium_log_collector:
+              name: cilium-log-collector
+              extraArgs: ''
+              archiveName: cilium-log-collector
+              archiveVersion: $(CILIUM_LOG_COLLECTOR_VERSION)
+              imageTag: $(Build.BuildNumber)
             cni:
               name: cni
               extraArgs: '--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID)'
@@ -206,6 +219,7 @@ stages:
       AZURE_IP_MASQ_MERGER_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureIpMasqMergerVersion'] ]
       AZURE_IPTABLES_MONITOR_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureIptablesMonitorVersion'] ]
       AZURE_BLOCK_IPTABLES_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.azureBlockIptablesVersion'] ]
+      CILIUM_LOG_COLLECTOR_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.ciliumLogCollectorVersion'] ]
       CNI_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.cniVersion'] ]
       CNS_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.cnsVersion'] ]
       IPV6_HP_BPF_VERSION: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.ipv6HpBpfVersion'] ]
@@ -221,6 +235,9 @@ stages:
       IPTABLES_MONITOR_LINUX_AMD64_REF: $(IMAGE_REPO_PATH)/linux-amd64/azure-iptables-monitor:$(Build.BuildNumber)
       IPTABLES_MONITOR_LINUX_ARM64_REF: $(IMAGE_REPO_PATH)/linux-arm64/azure-iptables-monitor:$(Build.BuildNumber)
 
+      CILIUM_LOG_COLLECTOR_LINUX_AMD64_REF: $(IMAGE_REPO_PATH)/linux-amd64/cilium-log-collector:$(Build.BuildNumber)
+      CILIUM_LOG_COLLECTOR_LINUX_ARM64_REF: $(IMAGE_REPO_PATH)/linux-arm64/cilium-log-collector:$(Build.BuildNumber)
+
       CNI_LINUX_AMD64_REF: $(IMAGE_REPO_PATH)/linux-amd64/cni:$(Build.BuildNumber)
       CNI_LINUX_ARM64_REF: $(IMAGE_REPO_PATH)/linux-arm64/cni:$(Build.BuildNumber)
       CNI_WINDOWS_AMD64_REF: $(IMAGE_REPO_PATH)/windows-amd64/cni:$(Build.BuildNumber)
@@ -269,6 +286,15 @@ stages:
               imageReference: $(IPTABLES_MONITOR_LINUX_AMD64_REF)
             - platform: linux/arm64
               imageReference: $(IPTABLES_MONITOR_LINUX_ARM64_REF)
+        - job: cilium_log_collector
+          templateContext:
+            name: cilium-log-collector
+            image_tag: $(CILIUM_LOG_COLLECTOR_VERSION)
+            platforms:
+            - platform: linux/amd64
+              imageReference: $(CILIUM_LOG_COLLECTOR_LINUX_AMD64_REF)
+            - platform: linux/arm64
+              imageReference: $(CILIUM_LOG_COLLECTOR_LINUX_ARM64_REF)
         - job: cni
           templateContext:
             name: cni

.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.stages.yaml

@@ -47,6 +47,8 @@ stages:
       IPAM_IMAGE_NAME_OVERRIDE: $(IMAGE_REPO_PATH)/azure-ipam
       CNS_IMAGE_NAME_OVERRIDE: $(IMAGE_REPO_PATH)/cns
       CNI_IMAGE_NAME_OVERRIDE: $(IMAGE_REPO_PATH)/cni
+      CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY_OVERRIDE: acnpublic.azurecr.io/$(IMAGE_REPO_PATH)
+      CILIUM_LOG_COLLECTOR_VERSION_TAG_OVERRIDE: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.ciliumLogCollectorVersion'] ]
       IPV6_HP_BPF_IMAGE_REPO_PATH: $(IMAGE_REPO_PATH)
     jobs:
       - job: ${{ parameters.name }}

.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml

@@ -42,8 +42,8 @@ steps:
         unset DIR
         unset CILIUM_VERSION_TAG
         make -C ./hack/aks deploy-cilium-dualstack
-        export CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY=acnpublic.azurecr.io
-        export CILIUM_LOG_COLLECTOR_VERSION_TAG=$(make cilium-log-collector-version)
+        export CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY=$(CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY_OVERRIDE)
+        export CILIUM_LOG_COLLECTOR_VERSION_TAG=$(CILIUM_LOG_COLLECTOR_VERSION_TAG_OVERRIDE)
         make -C ./hack/aks add-cilium-log-collector
         kubectl get po -owide -A
     name: "installCilium"

.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e.stages.yaml

@@ -47,6 +47,8 @@ stages:
       IPAM_IMAGE_NAME_OVERRIDE: $(IMAGE_REPO_PATH)/azure-ipam
       CNS_IMAGE_NAME_OVERRIDE: $(IMAGE_REPO_PATH)/cns
       CNI_IMAGE_NAME_OVERRIDE: $(IMAGE_REPO_PATH)/cni
+      CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY_OVERRIDE: acnpublic.azurecr.io/$(IMAGE_REPO_PATH)
+      CILIUM_LOG_COLLECTOR_VERSION_TAG_OVERRIDE: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.ciliumLogCollectorVersion'] ]
     condition: and(succeeded(), eq(variables.TAG, variables.CURRENT_VERSION))
     dependsOn:
     - setup

.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e.steps.yaml

@@ -55,8 +55,8 @@ steps:
         unset DIR
         unset CILIUM_VERSION_TAG
         make -C ./hack/aks deploy-cilium
-        export CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY=acnpublic.azurecr.io
-        export CILIUM_LOG_COLLECTOR_VERSION_TAG=$(make cilium-log-collector-version)
+        export CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY=$(CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY_OVERRIDE)
+        export CILIUM_LOG_COLLECTOR_VERSION_TAG=$(CILIUM_LOG_COLLECTOR_VERSION_TAG_OVERRIDE)
         make -C ./hack/aks add-cilium-log-collector
         kubectl get po -owide -A
         kubectl get crd -A

.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.stages.yaml

@@ -47,6 +47,8 @@ stages:
       IPAM_IMAGE_NAME_OVERRIDE: $(IMAGE_REPO_PATH)/azure-ipam
       CNS_IMAGE_NAME_OVERRIDE: $(IMAGE_REPO_PATH)/cns
       CNI_IMAGE_NAME_OVERRIDE: $(IMAGE_REPO_PATH)/cni
+      CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY_OVERRIDE: acnpublic.azurecr.io/$(IMAGE_REPO_PATH)
+      CILIUM_LOG_COLLECTOR_VERSION_TAG_OVERRIDE: $[ stageDependencies.setup.env.outputs['EnvironmentalVariables.ciliumLogCollectorVersion'] ]
     jobs:
       - job: ${{ parameters.name }}
         displayName: Cilium Overlay Test Suite - (${{ parameters.name }})

.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml

@@ -36,8 +36,8 @@ steps:
         unset DIR
         unset CILIUM_VERSION_TAG
         make -C ./hack/aks deploy-cilium-hubble
-        export CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY=acnpublic.azurecr.io
-        export CILIUM_LOG_COLLECTOR_VERSION_TAG=$(make cilium-log-collector-version)
+        export CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY=$(CILIUM_LOG_COLLECTOR_IMAGE_REGISTRY_OVERRIDE)
+        export CILIUM_LOG_COLLECTOR_VERSION_TAG=$(CILIUM_LOG_COLLECTOR_VERSION_TAG_OVERRIDE)
         make -C ./hack/aks add-cilium-log-collector
     name: "installCilium"
     displayName: "Install Cilium on AKS Overlay"