fix: App insight conflict resolve

.github/workflows/broken-links-checker.yml

@@ -37,7 +37,7 @@ jobs:
         uses: lycheeverse/lychee-action@v2.8.0
         with:
           args: >
-            --verbose --exclude-mail --no-progress --exclude ^https?://
+            --verbose --no-progress --exclude ^https?:// --exclude ^mailto:
             ${{ steps.changed-markdown-files.outputs.all_changed_files }}
           failIfEmpty: false
         env:
@@ -50,7 +50,7 @@ jobs:
         uses: lycheeverse/lychee-action@v2.8.0
         with:
           args: >
-            --verbose --exclude-mail --no-progress --exclude ^https?://
+            --verbose --no-progress --exclude ^https?:// --exclude ^mailto:
             '**/*.md'
           failIfEmpty: false
         env:

docs/LocalDevelopmentSetup.md

@@ -337,7 +337,7 @@ py -3.12 -m uv venv .venv
 py -3.12 -m uv sync --prerelease=allow
 ```
 
-> **⚠️ Important**: This repo currently depends on a prerelease/dev version of Microsoft Agent Framework. Always run `uv sync --prerelease=allow` (or `py -3.12 -m uv sync --prerelease=allow` on Windows) after creating the virtual environment to install all required dependencies. Missing dependencies will cause runtime errors like `ModuleNotFoundError: No module named 'pydantic'` or DNS resolution failures.
+> **⚠️ Important**: Always run `uv sync --prerelease=allow` (or `py -3.12 -m uv sync --prerelease=allow` on Windows) after creating the virtual environment to install all required dependencies. The `--prerelease=allow` flag is needed because some transitive dependencies may still use pre-release versions. Missing dependencies will cause runtime errors like `ModuleNotFoundError: No module named 'pydantic'` or DNS resolution failures.
 
 ### 5.4. Run the Processor
 

docs/ProcessFrameworkGuide.md

@@ -118,23 +118,21 @@ Inside each step, the orchestrator can use multi-agent patterns (maker-checker l
 
 - Implementation: [src/processor/src/steps/migration_processor.py](../src/processor/src/steps/migration_processor.py)
 - The processor creates a workflow with `WorkflowBuilder`.
-- It registers four executors, sets the start executor, and defines edges.
+- It instantiates four executors, passes the start executor to `WorkflowBuilder`, and chains them with `add_chain`.
 
 Example from the repo (simplified):
 
 ```python
 from agent_framework import WorkflowBuilder
 
+analysis_exec = AnalysisExecutor(id="analysis", app_context=app_context)
+design_exec = DesignExecutor(id="design", app_context=app_context)
+yaml_exec = YamlConvertExecutor(id="yaml", app_context=app_context)
+docs_exec = DocumentationExecutor(id="documentation", app_context=app_context)
+
 workflow = (
-    WorkflowBuilder()
-    .register_executor(lambda: AnalysisExecutor(id="analysis", app_context=app_context), name="analysis")
-    .register_executor(lambda: DesignExecutor(id="design", app_context=app_context), name="design")
-    .register_executor(lambda: YamlConvertExecutor(id="yaml", app_context=app_context), name="yaml")
-    .register_executor(lambda: DocumentationExecutor(id="documentation", app_context=app_context), name="documentation")
-    .set_start_executor("analysis")
-    .add_edge("analysis", "design")
-    .add_edge("design", "yaml")
-    .add_edge("yaml", "documentation")
+    WorkflowBuilder(start_executor=analysis_exec)
+    .add_chain([analysis_exec, design_exec, yaml_exec, docs_exec])
     .build()
 )
 ```
@@ -355,7 +353,7 @@ To run processor unit tests locally (example):
 
 ```bash
 cd src/processor
-uv run --prerelease=allow python -m pytest src/processor/src/tests/unit -v
+uv run python -m pytest src/tests/unit -v
 ```
 
 ## Extending the pipeline

infra/main.bicep

@@ -301,8 +301,12 @@ module applicationInsights 'br/public:avm/res/insights/component:0.6.0' = if (en
     disableIpMasking: false
     flowType: 'Bluefield'
     // WAF aligned configuration for Monitoring
+    // The AVM `insights/component` module wires Application Insights to
+    // the Log Analytics workspace via `workspaceResourceId` (workspace-
+    // based App Insights). A separate `diagnosticSettings` entry on the
+    // SAME workspace causes duplicate ingestion of platform logs.
+    // Source: AB#37816 — see CKM #811 reference implementation.
     workspaceResourceId: enableMonitoring ? logAnalyticsWorkspaceResourceId : ''
-    diagnosticSettings: enableMonitoring ? [{ workspaceResourceId: logAnalyticsWorkspaceResourceId }] : null
   }
 }
 

infra/main_custom.bicep

@@ -279,8 +279,12 @@ module applicationInsights 'br/public:avm/res/insights/component:0.6.0' = if (en
     disableIpMasking: false
     flowType: 'Bluefield'
     // WAF aligned configuration for Monitoring
+    // The AVM `insights/component` module wires Application Insights to
+    // the Log Analytics workspace via `workspaceResourceId` (workspace-
+    // based App Insights). A separate `diagnosticSettings` entry on the
+    // SAME workspace causes duplicate ingestion of platform logs.
+    // Source: AB#37816 — see CKM #811 reference implementation.
     workspaceResourceId: enableMonitoring ? logAnalyticsWorkspaceResourceId : ''
-    diagnosticSettings: enableMonitoring ? [{ workspaceResourceId: logAnalyticsWorkspaceResourceId }] : null
   }
 }
 

infra/vscode_web/codeSample.py

@@ -1,3 +1,4 @@
+from azure.ai.agents.models import ListSortOrder
 from azure.ai.projects import AIProjectClient
 from azure.identity import DefaultAzureCredential
 
@@ -7,19 +8,25 @@
 
 agent = project_client.agents.get_agent("<%= agentId %>")
 
-thread = project_client.agents.create_thread()
+thread = project_client.agents.threads.create()
 print(f"Created thread, ID: {thread.id}")
 
-message = project_client.agents.create_message(
+message = project_client.agents.messages.create(
     thread_id=thread.id,
     role="user",
     content="<%= userMessage %>"
 )
 
-run = project_client.agents.create_and_process_run(
+run = project_client.agents.runs.create_and_process(
     thread_id=thread.id,
     agent_id=agent.id)
-messages = project_client.agents.list_messages(thread_id=thread.id)
 
-for text_message in messages.text_messages:
-    print(text_message.as_dict())
+if run.status == "failed":
+    print(f"Run failed: {run.last_error}")
+else:
+    messages = project_client.agents.messages.list(
+        thread_id=thread.id, order=ListSortOrder.ASCENDING)
+
+    for message in messages:
+        if message.text_messages:
+            print(f"{message.role}: {message.text_messages[-1].text.value}")

infra/vscode_web/requirements.txt

@@ -1,3 +1,3 @@
-azure-ai-projects==1.0.0b12
+azure-ai-projects==2.1.0
 azure-identity==1.20.0
 ansible-core~=2.17.0

src/backend-api/pyproject.toml

@@ -9,12 +9,14 @@ dependencies = [
     "azure-ai-agents==1.2.0b6",
     "azure-appconfiguration==1.8.0",
     "azure-identity==1.25.3",
+    "azure-monitor-events-extension==0.1.0",
     "azure-monitor-opentelemetry==1.8.7",
     "azure-search-documents==11.7.0b2",
     "azure-storage-blob==12.28.0",
     "azure-storage-queue==12.15.0",
     "fastapi[standard]==0.135.3",
     "httpx==0.28.1",
+    "opentelemetry-instrumentation-fastapi==0.61b0",
     "pydantic-settings==2.13.1",
     "python-dotenv==1.2.2",
     "python-multipart==0.0.27",

src/backend-api/src/app/.env.example

@@ -7,3 +7,17 @@ APP_CONFIGURATION_URL=""
 # AZURE_PACKAGE_LOGGING_LEVEL="WARNING"  # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL
 # AZURE_LOGGING_PACKAGES="azure.core.pipeline.policies.http_logging_policy,azure.storage.blob,azure.storage.queue,azure.core,azure.identity,azure.storage,azure.core.pipeline,azure.core.pipeline.policies,azure.core.pipeline.transport,openai,openai._client,httpx,httpcore,semantic_kernel,urllib3,msal"
 
+# ------------------------------------------------------------------
+# Application Insights / OpenTelemetry
+# ------------------------------------------------------------------
+# When deployed via the bundled Bicep, this value is injected by
+# `infra/main.bicep` -> `containerAppBackend` from the
+# `applicationInsights.outputs.connectionString` output. Leave unset for
+# local dev to skip telemetry export entirely (the app will log a single
+# warning at startup and otherwise behave normally).
+# APPLICATIONINSIGHTS_CONNECTION_STRING=""
+
+# Optional: clamp basic logging level for App Insights ingestion.
+# Defaults inherit from APP_LOGGING_LEVEL above.
+# AZURE_BASIC_LOGGING_LEVEL="INFO"
+

src/backend-api/src/app/application.py

@@ -1,9 +1,14 @@
+import logging
 import os
 from datetime import datetime
 
 from fastapi.middleware.cors import CORSMiddleware
 from libs.base.application_base import Application_Base
 from libs.base.typed_fastapi import TypedFastAPI
+from libs.logging.span_filters import (
+    DropASGIResponseBodySpanProcessor,
+    DropCosmosDependencySpanProcessor,
+)
 from libs.repositories.file_repository import FileRepository
 from libs.repositories.process_repository import ProcessRepository
 from libs.repositories.process_status_repository import ProcessStatusRepository
@@ -20,6 +25,14 @@
 from routers import router_debug, router_files, router_process
 from routers.http_probes import router as http_probes
 
+logger = logging.getLogger(__name__)
+
+# URLs (relative paths) that should NOT generate request telemetry.
+# Matches the routes registered in `routers/http_probes.py`.
+# `excluded_urls` is a comma-separated substring list per the
+# OpenTelemetry FastAPI instrumentation contract.
+_OTEL_EXCLUDED_URLS = "health,startup"
+
 
 class Application(Application_Base):
     """
@@ -57,11 +70,94 @@ def initialize(self):
             allow_headers=["*"],
         )
 
+        # Wire up Azure Monitor / OpenTelemetry BEFORE the routers are
+        # included so that the FastAPI instrumentor can patch the app
+        # while the route table is still empty. `configure_azure_monitor`
+        # is a no-op (warns once) when the connection string env var is
+        # absent — see `libs/logging/event_utils.py`.
+        self._configure_azure_monitor()
+
         self.app.include_router(http_probes)
         self._register_dependencies()
         self._config_routers()
         # self._initialize_database()
 
+        # Instrumenting AFTER routers are registered means every route is
+        # automatically wrapped by the OTEL middleware.
+        self._instrument_fastapi()
+
+    def _configure_azure_monitor(self):
+        """Initialise Azure Monitor OpenTelemetry exporter, if configured.
+
+        This is the App Insights "linkage" step required by AC #1 / AC #2:
+        if `APPLICATIONINSIGHTS_CONNECTION_STRING` is set in the
+        container app environment (wired by Bicep — see
+        `infra/main.bicep`), we hand it to `configure_azure_monitor`
+        along with our two noise-suppressing span processors.
+
+        Live Metrics is enabled so the team can watch the Maintenance
+        environment in real time during demo validation.
+        """
+        connection_string = os.environ.get(
+            "APPLICATIONINSIGHTS_CONNECTION_STRING", ""
+        ).strip()
+        if not connection_string:
+            logger.info(
+                "APPLICATIONINSIGHTS_CONNECTION_STRING not set; "
+                "skipping Azure Monitor OpenTelemetry configuration."
+            )
+            return
+
+        try:
+            from azure.monitor.opentelemetry import configure_azure_monitor
+
+            configure_azure_monitor(
+                connection_string=connection_string,
+                enable_live_metrics=True,
+                span_processors=[
+                    DropASGIResponseBodySpanProcessor(),
+                    DropCosmosDependencySpanProcessor(),
+                ],
+            )
+            # Do NOT log the connection string itself — it contains the
+            # ingestion key. Logging only the fact of configuration.
+            logger.info(
+                "Azure Monitor OpenTelemetry configured (live metrics enabled)."
+            )
+        except Exception:  # noqa: BLE001 — telemetry must never break startup
+            logger.exception(
+                "Failed to configure Azure Monitor OpenTelemetry; "
+                "continuing without App Insights export."
+            )
+
+    def _instrument_fastapi(self):
+        """Apply the OpenTelemetry FastAPI instrumentation.
+
+        Excludes the liveness / startup probe routes (registered in
+        `routers/http_probes.py`) so probe traffic does not flood
+        Application Insights with no-information request rows.
+        """
+        if not os.environ.get(
+            "APPLICATIONINSIGHTS_CONNECTION_STRING", ""
+        ).strip():
+            # No exporter wired up; instrumenting is a wasted import.
+            return
+        try:
+            from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+
+            FastAPIInstrumentor.instrument_app(
+                self.app, excluded_urls=_OTEL_EXCLUDED_URLS
+            )
+            logger.info(
+                "FastAPIInstrumentor attached (excluded_urls=%s).",
+                _OTEL_EXCLUDED_URLS,
+            )
+        except Exception:  # noqa: BLE001
+            logger.exception(
+                "Failed to attach FastAPIInstrumentor; "
+                "continuing without per-request telemetry."
+            )
+
     def _config_routers(self):
         """
         Configure routers for the FastAPI application.

src/backend-api/src/app/libs/base/application_base.py

@@ -15,6 +15,19 @@
 from libs.application.application_context import AppContext
 from libs.azure.app_configuration import AppConfigurationHelper
 
+# Logger packages that emit at INFO/DEBUG levels often enough to drown
+# out signal in Application Insights logs when the OTEL log handler is
+# attached. We force them to WARNING regardless of caller config.
+# This is a backstop in addition to (not a replacement for) the
+# `AZURE_LOGGING_PACKAGES` env-var driven filtering — anything listed
+# here is ALWAYS clamped to WARNING.
+_NOISY_LOGGER_PACKAGES = (
+    "azure.core.pipeline.policies.http_logging_policy",
+    "azure.cosmos",
+    "opentelemetry.sdk",
+    "azure.monitor.opentelemetry.exporter.export._base",
+)
+
 
 class Application_Base(ABC):
     application_context: AppContext = None
@@ -78,6 +91,17 @@ def __init__(self, env_file_path: str | None = None, **data):
                 ):
                     logging.getLogger(logger_name).setLevel(azure_level)
 
+            # Hard-suppress known noisy packages regardless of operator
+            # config. Without this, the App Insights logs view is
+            # dominated by per-request HTTP policy logs and per-call
+            # Cosmos diagnostics — see AC #3 / AC #4 of AB#37816.
+            # We never lower a logger that the operator has explicitly
+            # raised below WARNING.
+            for noisy_pkg in _NOISY_LOGGER_PACKAGES:
+                noisy_logger = logging.getLogger(noisy_pkg)
+                if noisy_logger.level == logging.NOTSET or noisy_logger.level < logging.WARNING:
+                    noisy_logger.setLevel(logging.WARNING)
+
         # Initialize the application
         self.initialize()
 

src/backend-api/src/app/libs/logging/__init__.py

@@ -0,0 +1,23 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+"""Application-level logging and telemetry helpers.
+
+This subpackage hosts the small Application Insights / OpenTelemetry
+integration helpers used by the backend-api application:
+
+- ``event_utils`` — a tiny wrapper around
+  ``azure.monitor.events.extension.track_event`` that no-ops when the
+  ``APPLICATIONINSIGHTS_CONNECTION_STRING`` environment variable is not
+  configured. Callers can therefore emit structured events from
+  any router/service without conditionally guarding each call site.
+- ``span_filters`` — custom OpenTelemetry ``SpanProcessor`` implementations
+  that drop noisy spans before they are exported to Application Insights
+  (per-chunk ASGI ``http.response.body`` spans and Cosmos DB dependency
+  spans). These keep the App Insights ingestion cost and the
+  end-to-end transaction view clean for the Container Migration workflow.
+
+Nothing in this subpackage imports Azure SDKs at module-import time, so
+it is safe to import from contexts where the App Insights SDK may not be
+fully wired up yet (e.g. application bootstrap before
+``configure_azure_monitor`` has run).
+"""