diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 18382cf5..e043a359 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -25,3 +25,11 @@ jobs: run: | cd aws_quickstart python -B -S -m unittest attach_integration_permissions_test.py -v + - name: Run shared CloudFormation helper unit tests + run: | + cd aws_quickstart + python -B -S -m unittest cfn_common_test.py -v + - name: Run Operator subscription unit tests + run: | + cd aws_quickstart + python -B -S -m unittest accept_operator_subscription_test.py -v diff --git a/aws_quickstart/CHANGELOG.md b/aws_quickstart/CHANGELOG.md index f64d8d14..2d00d62c 100644 --- a/aws_quickstart/CHANGELOG.md +++ b/aws_quickstart/CHANGELOG.md @@ -1,3 +1,7 @@ +# 4.19.0 (August 21, 2026) + +- Accept the free Datadog Operator AWS Marketplace agreement in the commercial AWS partition when EKS instrumentation is selected, allowing the managed add-on installation to proceed without Marketplace permissions on the Datadog integration role. GovCloud and China deployments skip automatic agreement acceptance. + # 4.18.1 (July 31, 2026) - Track every CloudFormation stack using an instrumenter permissions-boundary policy and safely remove the policy after the final owning stack releases it and no IAM entities remain. diff --git a/aws_quickstart/accept_operator_subscription.py b/aws_quickstart/accept_operator_subscription.py new file mode 100644 index 00000000..1dca8c77 --- /dev/null +++ b/aws_quickstart/accept_operator_subscription.py @@ -0,0 +1,719 @@ +import json +import logging +import time +import uuid +from datetime import datetime, timezone +from decimal import Decimal, InvalidOperation + +import boto3 +from botocore.config import Config +import cfnresponse + +from cfn_common import send_cfn_response + + +LOGGER = logging.getLogger() +LOGGER.setLevel(logging.INFO) + +DATADOG_OPERATOR_PRODUCT_ID = "6e852b2a-ecbb-431c-9b63-7de0288f4d00" +MARKETPLACE_CATALOG = "AWSMarketplace" +MARKETPLACE_REGION = "us-east-1" +ENTITLEMENT_ATTEMPTS = 24 +ENTITLEMENT_DELAY_SECONDS = 5 +SDK_CONNECT_TIMEOUT_SECONDS = 5 +SDK_READ_TIMEOUT_SECONDS = 10 +SDK_REQUEST_BUDGET_SECONDS = 32 +CLOUDFORMATION_RESPONSE_BUFFER_SECONDS = 15 + +AWS_CONFIG = Config( + retries={"total_max_attempts": 2, "mode": "standard"}, + connect_timeout=SDK_CONNECT_TIMEOUT_SECONDS, + read_timeout=SDK_READ_TIMEOUT_SECONDS, +) + + +class SubscriptionError(Exception): + def __init__( + self, + stage, + reason, + message, + *, + offer_id=None, + agreement_request_id=None, + agreement_id=None, + ): + super().__init__(message) + self.stage = stage + self.reason = reason + self.offer_id = offer_id + self.agreement_request_id = agreement_request_id + self.agreement_id = agreement_id + + +def _log(stage, result, reason, **fields): + payload = { + "marketplace_stage": stage, + "marketplace_result": result, + "marketplace_reason": reason, + "marketplace_control_plane_region": MARKETPLACE_REGION, + **{key: value for key, value in fields.items() if value is not None}, + } + LOGGER.info( + "Datadog Operator Marketplace subscription event %s", + json.dumps(payload, default=str, sort_keys=True), + ) + + +def _require_sdk_request_budget(deadline, operation): + if ( + deadline is not None + and time.monotonic() + SDK_REQUEST_BUDGET_SECONDS >= deadline + ): + raise RuntimeError( + f"{operation} was not attempted because the Lambda deadline is near" + ) + + +def _pages( + client, + operation, + result_key, + *, + error_stage, + error_message, + deadline=None, + **kwargs, +): + try: + paginator = iter(client.get_paginator(operation).paginate(**kwargs)) + while True: + _require_sdk_request_budget(deadline, operation) + try: + page = next(paginator) + except StopIteration: + return + yield from page.get(result_key, []) + except SubscriptionError: + raise + except Exception as error: + raise SubscriptionError( + error_stage, + "aws_api_error", + f"{error_message}: {error}", + ) from error + + +def _api_call(stage, message, operation, *, deadline=None, **kwargs): + try: + _require_sdk_request_budget( + deadline, + getattr(operation, "__name__", "AWS API call"), + ) + return operation(**kwargs) + except Exception as error: + raise SubscriptionError(stage, "aws_api_error", f"{message}: {error}") from error + + +def find_active_agreement(agreement_client, *, deadline=None): + filters = [ + {"name": "PartyType", "values": ["Acceptor"]}, + {"name": "AgreementType", "values": ["PurchaseAgreement"]}, + { + "name": "ResourceIdentifier", + "values": [DATADOG_OPERATOR_PRODUCT_ID], + }, + {"name": "Status", "values": ["ACTIVE"]}, + ] + agreements = list( + _pages( + agreement_client, + "search_agreements", + "agreementViewSummaries", + error_stage="agreement_discovery", + error_message="Failed to search for an active Datadog Operator Marketplace agreement", + deadline=deadline, + catalog=MARKETPLACE_CATALOG, + filters=filters, + ) + ) + if not agreements: + return None + if len(agreements) != 1: + raise SubscriptionError( + "agreement_discovery", + "multiple_active_agreements", + "Multiple active Datadog Operator Marketplace agreements were returned", + ) + + agreement_id = agreements[0].get("agreementId") + if not agreement_id: + raise SubscriptionError( + "agreement_discovery", + "invalid_response", + "The active Datadog Operator Marketplace agreement has no identifier", + ) + return agreement_id + + +def _is_available(resource, now): + available_from = resource.get("availableFromTime") + expiration = resource.get("expirationTime") + return (available_from is None or now >= available_from) and ( + expiration is None or now < expiration + ) + + +def _validate_purchase_option(option): + offer_id = option.get("purchaseOptionId") + if not offer_id or option.get("purchaseOptionType") != "OFFER": + raise SubscriptionError( + "offer_discovery", + "invalid_offer", + "The Datadog Operator Marketplace purchase option is not an offer", + offer_id=offer_id, + ) + + entities = option.get("associatedEntities", []) + if len(entities) != 1: + raise SubscriptionError( + "offer_discovery", + "invalid_offer", + "The Datadog Operator Marketplace purchase option has unexpected associated entities", + offer_id=offer_id, + ) + entity = entities[0] + if ( + entity.get("product", {}).get("productId") + != DATADOG_OPERATOR_PRODUCT_ID + or entity.get("offer", {}).get("offerId") != offer_id + ): + raise SubscriptionError( + "offer_discovery", + "invalid_offer", + "The Datadog Operator Marketplace purchase option does not match " + "the expected product and offer", + offer_id=offer_id, + ) + return offer_id + + +def _validate_offer(offer_id, offer): + entities = offer.get("associatedEntities", []) + if ( + offer.get("offerId") != offer_id + or not offer.get("agreementProposalId") + or len(entities) != 1 + or entities[0].get("product", {}).get("productId") + != DATADOG_OPERATOR_PRODUCT_ID + or not offer.get("pricingModel") + ): + raise SubscriptionError( + "offer_discovery", + "invalid_offer", + "The Datadog Operator Marketplace offer details are incomplete " + "or do not match the expected product", + offer_id=offer_id, + ) + + +def find_free_offer(discovery_client, now=None, *, deadline=None): + now = now or datetime.now(timezone.utc) + candidates = _pages( + discovery_client, + "list_purchase_options", + "purchaseOptions", + error_stage="offer_discovery", + error_message="Failed to list Datadog Operator Marketplace purchase options", + deadline=deadline, + filters=[ + { + "filterType": "PRODUCT_ID", + "filterValues": [DATADOG_OPERATOR_PRODUCT_ID], + }, + {"filterType": "PURCHASE_OPTION_TYPE", "filterValues": ["OFFER"]}, + ], + ) + + free_offers = [] + candidate_count = 0 + unavailable_count = 0 + ineligible_count = 0 + for option in candidates: + candidate_count += 1 + offer_id = _validate_purchase_option(option) + if not _is_available(option, now): + unavailable_count += 1 + continue + + offer = _api_call( + "offer_discovery", + f"Failed to get Datadog Operator Marketplace offer {offer_id}", + discovery_client.get_offer, + deadline=deadline, + offerId=offer_id, + ) + _validate_offer(offer_id, offer) + if not _is_available(offer, now): + unavailable_count += 1 + continue + if ( + offer["pricingModel"].get("pricingModelType") != "FREE" + or offer.get("badges") + ): + ineligible_count += 1 + continue + free_offers.append(offer) + + if not free_offers: + raise SubscriptionError( + "offer_discovery", + "no_eligible_free_offer", + "No public free Datadog Operator Marketplace offer was returned " + f"(candidates={candidate_count} unavailable={unavailable_count} " + f"nonfree_or_badged={ineligible_count})", + ) + if len(free_offers) != 1: + raise SubscriptionError( + "offer_discovery", + "multiple_free_offers", + "Multiple public free Datadog Operator Marketplace offers were returned", + ) + return free_offers[0] + + +def requested_terms(discovery_client, offer_id, *, deadline=None): + terms = list( + _pages( + discovery_client, + "get_offer_terms", + "offerTerms", + error_stage="offer_terms", + error_message=f"Failed to get Datadog Operator Marketplace offer {offer_id} terms", + deadline=deadline, + offerId=offer_id, + ) + ) + term_ids_by_name = {} + supported = {"legalTerm", "supportTerm"} + for term in terms: + present = [name for name in supported if name in term] + if len(present) != 1 or len(term) != 1: + raise SubscriptionError( + "offer_terms", + "unsupported_terms", + "The free Datadog Operator Marketplace offer contains an unsupported term", + offer_id=offer_id, + ) + term_name = present[0] + if term_name in term_ids_by_name: + raise SubscriptionError( + "offer_terms", + "invalid_terms", + f"The Datadog Operator Marketplace offer contains multiple {term_name} values", + offer_id=offer_id, + ) + term_id = term[term_name].get("id") + if not term_id: + raise SubscriptionError( + "offer_terms", + "invalid_terms", + "The Datadog Operator Marketplace offer contains a term without an identifier", + offer_id=offer_id, + ) + term_ids_by_name[term_name] = term_id + + if term_ids_by_name.keys() != supported: + raise SubscriptionError( + "offer_terms", + "invalid_terms", + "The Datadog Operator Marketplace offer must contain exactly one " + "legal and one support term", + offer_id=offer_id, + ) + return [{"id": term_id} for term_id in sorted(term_ids_by_name.values())] + + +def _require_zero(value, field, *, required=False): + if value is None: + if required: + raise SubscriptionError( + "quote_validation", + "unknown_quote_amount", + f"The free Datadog Operator Marketplace quote has no value for {field}", + ) + return + try: + amount = Decimal(value) + except (InvalidOperation, TypeError, ValueError) as error: + raise SubscriptionError( + "quote_validation", + "unknown_quote_amount", + f"The free Datadog Operator Marketplace quote has an invalid value for {field}", + ) from error + if amount != 0: + raise SubscriptionError( + "quote_validation", + "nonzero_quote", + f"The free Datadog Operator Marketplace quote contains a nonzero charge at {field}", + ) + + +def _validate_amount_fields(node, path=""): + if isinstance(node, dict): + for key, value in node.items(): + field = f"{path}.{key}" if path else key + if isinstance(value, (dict, list)): + _validate_amount_fields(value, field) + elif key.lower().endswith(("amount", "amountaftertax")): + _require_zero(value, field) + elif isinstance(node, list): + for index, item in enumerate(node): + _validate_amount_fields(item, f"{path}[{index}]") + + +def validate_zero_charge_summary(summary): + if summary is None: + raise SubscriptionError( + "quote_validation", + "unknown_quote_amount", + "The free Datadog Operator Marketplace agreement quote has no charge summary", + ) + + _require_zero( + summary.get("newAgreementValue"), + "newAgreementValue", + required=True, + ) + _require_zero( + summary.get("newAgreementValueAfterTax"), + "newAgreementValueAfterTax", + ) + for index, charge in enumerate(summary.get("expectedCharges", [])): + _require_zero( + charge.get("amount"), + f"expectedCharges[{index}].amount", + required=True, + ) + for index, charge in enumerate(summary.get("itemizedCharges", [])): + _require_zero( + charge.get("incrementalChargeAmount"), + f"itemizedCharges[{index}].incrementalChargeAmount", + required=True, + ) + _validate_amount_fields(summary) + + +def _client_token(event, proposal_id, terms): + seed = "\0".join( + [ + event["StackId"], + event["LogicalResourceId"], + event["RequestId"], + proposal_id, + *(term["id"] for term in terms), + ] + ) + return str(uuid.uuid5(uuid.NAMESPACE_OID, seed)) + + +def create_and_accept_agreement( + event, + discovery_client, + agreement_client, + *, + deadline=None, +): + offer = find_free_offer(discovery_client, deadline=deadline) + offer_id = offer["offerId"] + terms = requested_terms(discovery_client, offer_id, deadline=deadline) + response = _api_call( + "request_creation", + "Failed to create the Datadog Operator Marketplace agreement request", + agreement_client.create_agreement_request, + deadline=deadline, + agreementProposalIdentifier=offer["agreementProposalId"], + clientToken=_client_token(event, offer["agreementProposalId"], terms), + intent="NEW", + requestedTerms=terms, + ) + try: + validate_zero_charge_summary(response.get("chargeSummary")) + except SubscriptionError as error: + error.offer_id = offer_id + raise + agreement_request_id = response.get("agreementRequestId") + if not agreement_request_id: + raise SubscriptionError( + "request_creation", + "invalid_response", + "The Datadog Operator Marketplace agreement request has no identifier", + offer_id=offer_id, + ) + _log( + "request_creation", + "succeeded", + "free_quote_validated", + marketplace_offer_id=offer_id, + marketplace_agreement_request_id=agreement_request_id, + ) + + _require_sdk_request_budget(deadline, "accept_agreement_request") + try: + accepted = agreement_client.accept_agreement_request( + agreementRequestId=agreement_request_id + ) + except Exception as acceptance_error: + try: + agreement_id = find_active_agreement( + agreement_client, + deadline=deadline, + ) + except Exception as recovery_error: + raise SubscriptionError( + "acceptance_recovery", + "recovery_failed", + "Failed to accept the Datadog Operator Marketplace agreement " + f"request and could not determine whether it succeeded: {recovery_error}", + offer_id=offer_id, + agreement_request_id=agreement_request_id, + ) from acceptance_error + if agreement_id: + _log( + "request_acceptance", + "succeeded", + "active_agreement_recovered", + marketplace_offer_id=offer_id, + marketplace_agreement_request_id=agreement_request_id, + marketplace_agreement_id=agreement_id, + ) + return agreement_id + raise SubscriptionError( + "request_acceptance", + "aws_api_error", + "Failed to accept the Datadog Operator Marketplace agreement " + f"request: {acceptance_error}", + offer_id=offer_id, + agreement_request_id=agreement_request_id, + ) from acceptance_error + + agreement_id = accepted.get("agreementId") + if not agreement_id: + raise SubscriptionError( + "request_acceptance", + "invalid_response", + "The accepted Datadog Operator Marketplace agreement has no identifier", + offer_id=offer_id, + agreement_request_id=agreement_request_id, + ) + _log( + "request_acceptance", + "succeeded", + "agreement_accepted", + marketplace_offer_id=offer_id, + marketplace_agreement_request_id=agreement_request_id, + marketplace_agreement_id=agreement_id, + ) + return agreement_id + + +def entitlement_status(agreement_client, agreement_id, *, deadline=None): + matches = [ + entitlement + for entitlement in _pages( + agreement_client, + "get_agreement_entitlements", + "agreementEntitlements", + error_stage="entitlement", + error_message=( + "Failed to get Datadog Operator Marketplace agreement " + f"{agreement_id} entitlements" + ), + deadline=deadline, + agreementId=agreement_id, + ) + if entitlement.get("resource", {}).get("id") + == DATADOG_OPERATOR_PRODUCT_ID + ] + if len(matches) > 1: + raise SubscriptionError( + "entitlement", + "invalid_response", + "Multiple Datadog Operator Marketplace entitlements were returned", + agreement_id=agreement_id, + ) + return matches[0] if matches else None + + +def wait_for_entitlement( + agreement_client, + agreement_id, + *, + attempts=ENTITLEMENT_ATTEMPTS, + delay=ENTITLEMENT_DELAY_SECONDS, + deadline=None, +): + last_status = None + last_reason = None + for attempt in range(attempts): + if ( + deadline is not None + and time.monotonic() + SDK_REQUEST_BUDGET_SECONDS >= deadline + ): + break + entitlement = entitlement_status( + agreement_client, agreement_id, deadline=deadline + ) + status = entitlement.get("status") if entitlement else None + reason = entitlement.get("statusReasonCode") if entitlement else None + last_status = status + last_reason = reason + if status == "PROVISIONED": + _log( + "entitlement", + "succeeded", + "entitlement_provisioned", + marketplace_agreement_id=agreement_id, + marketplace_entitlement_status=status, + marketplace_entitlement_reason=reason, + ) + return + if status in {"FAILED", "DEPROVISIONED"}: + raise SubscriptionError( + "entitlement", + "entitlement_failed", + f"The Datadog Operator Marketplace entitlement is {status} ({reason})", + agreement_id=agreement_id, + ) + if status not in {None, "PENDING", "SCHEDULED"}: + raise SubscriptionError( + "entitlement", + "unsupported_entitlement_status", + f"The Datadog Operator Marketplace entitlement has unsupported status {status}", + agreement_id=agreement_id, + ) + if attempt + 1 < attempts: + if ( + deadline is not None + and time.monotonic() + delay + SDK_REQUEST_BUDGET_SECONDS >= deadline + ): + break + time.sleep(delay) + + raise SubscriptionError( + "entitlement", + "entitlement_timeout", + "Timed out waiting for the Datadog Operator Marketplace entitlement " + f"(status={last_status} reason={last_reason})", + agreement_id=agreement_id, + ) + + +def ensure_subscription(event, *, deadline=None): + try: + session = boto3.Session() + discovery_client = session.client( + "marketplace-discovery", + region_name=MARKETPLACE_REGION, + config=AWS_CONFIG, + ) + agreement_client = session.client( + "marketplace-agreement", + region_name=MARKETPLACE_REGION, + config=AWS_CONFIG, + ) + except Exception as error: + raise SubscriptionError( + "sdk_initialization", + "unsupported_sdk", + "The Lambda runtime AWS SDK could not initialize the required Marketplace " + f"clients (boto3={getattr(boto3, '__version__', 'unknown')}): {error}", + ) from error + _log( + "sdk_initialization", + "succeeded", + "clients_created", + boto3_version=getattr(boto3, "__version__", "unknown"), + ) + + agreement_id = find_active_agreement(agreement_client, deadline=deadline) + if agreement_id: + _log( + "agreement_discovery", + "succeeded", + "active_agreement_reused", + marketplace_agreement_id=agreement_id, + ) + else: + _log( + "agreement_discovery", + "succeeded", + "active_agreement_not_found", + ) + agreement_id = create_and_accept_agreement( + event, + discovery_client, + agreement_client, + deadline=deadline, + ) + wait_for_entitlement(agreement_client, agreement_id, deadline=deadline) + return agreement_id + + +def handler(event, context): + request_type = event["RequestType"] + properties = event["ResourceProperties"] + account_id = properties.get("AccountId") + if request_type == "Delete": + _log( + "cloudformation_delete", + "succeeded", + "agreement_retained", + account_id=account_id, + ) + send_cfn_response( + cfnresponse, + event, + context, + cfnresponse.SUCCESS, + {"AgreementRetained": True}, + ) + return + + try: + remaining_seconds = context.get_remaining_time_in_millis() / 1000 + deadline = time.monotonic() + max( + 0, + remaining_seconds - CLOUDFORMATION_RESPONSE_BUFFER_SECONDS, + ) + agreement_id = ensure_subscription(event, deadline=deadline) + send_cfn_response( + cfnresponse, + event, + context, + cfnresponse.SUCCESS, + {"AgreementId": agreement_id}, + ) + except Exception as error: + stage = getattr(error, "stage", "subscription") + reason = getattr(error, "reason", "aws_api_error") + _log( + stage, + "failed", + reason, + account_id=account_id, + marketplace_offer_id=getattr(error, "offer_id", None), + marketplace_agreement_request_id=getattr( + error, "agreement_request_id", None + ), + marketplace_agreement_id=getattr(error, "agreement_id", None), + error=str(error), + ) + LOGGER.exception("Failed to accept the Datadog Operator Marketplace agreement") + send_cfn_response( + cfnresponse, + event, + context, + cfnresponse.FAILED, + {"Message": str(error)}, + ) diff --git a/aws_quickstart/accept_operator_subscription_test.py b/aws_quickstart/accept_operator_subscription_test.py new file mode 100644 index 00000000..7d135950 --- /dev/null +++ b/aws_quickstart/accept_operator_subscription_test.py @@ -0,0 +1,730 @@ +#!/usr/bin/env python3 + +from datetime import datetime, timedelta, timezone +from pathlib import Path +import sys +import unittest +from unittest.mock import MagicMock, patch + + +if "boto3" not in sys.modules: + sys.modules["boto3"] = MagicMock() +if "botocore.config" not in sys.modules: + sys.modules["botocore"] = MagicMock() + sys.modules["botocore.config"] = MagicMock() +if "cfnresponse" not in sys.modules: + cfnresponse = MagicMock() + cfnresponse.SUCCESS = "SUCCESS" + cfnresponse.FAILED = "FAILED" + sys.modules["cfnresponse"] = cfnresponse + + +from accept_operator_subscription import ( + DATADOG_OPERATOR_PRODUCT_ID, + SubscriptionError, + create_and_accept_agreement, + entitlement_status, + ensure_subscription, + find_active_agreement, + find_free_offer, + handler, + requested_terms, + validate_zero_charge_summary, + wait_for_entitlement, +) + + +def paginator_client(**operation_pages): + client = MagicMock() + paginators = {} + for operation, pages in operation_pages.items(): + paginator = MagicMock() + paginator.paginate.return_value = pages + paginators[operation] = paginator + client.get_paginator.side_effect = paginators.__getitem__ + client.paginators = paginators + return client + + +def purchase_option(offer_id="offer-1", **overrides): + value = { + "purchaseOptionId": offer_id, + "purchaseOptionType": "OFFER", + "associatedEntities": [ + { + "product": {"productId": DATADOG_OPERATOR_PRODUCT_ID}, + "offer": {"offerId": offer_id}, + } + ], + } + value.update(overrides) + return value + + +def free_offer(offer_id="offer-1", **overrides): + value = { + "offerId": offer_id, + "agreementProposalId": "ap-proposal1", + "associatedEntities": [ + {"product": {"productId": DATADOG_OPERATOR_PRODUCT_ID}} + ], + "pricingModel": {"pricingModelType": "FREE"}, + "badges": [], + } + value.update(overrides) + return value + + +def event(request_type="Create"): + return { + "RequestType": request_type, + "RequestId": "request-1", + "StackId": "arn:aws:cloudformation:us-east-1:123456789012:stack/test/id", + "LogicalResourceId": "DatadogOperatorSubscriptionFunctionTrigger", + "ResourceProperties": { + "AccountId": "123456789012", + }, + } + + +class TestTemplate(unittest.TestCase): + def test_template_embeds_subscription_source(self): + template = Path(__file__).with_name( + "datadog_integration_permissions.yaml" + ).read_text() + + self.assertEqual(template.count(""), 1) + self.assertIn( + "Conditions:\n" + " IncludeEKS:\n" + " Fn::And:\n" + " - Fn::Equals:\n" + " - !Ref AWS::Partition\n" + " - aws\n" + " - Fn::Not:\n", + template, + ) + self.assertIn( + " InstrumentationResourceTypes:\n Type: CommaDelimitedList", + template, + ) + self.assertEqual( + template.count( + 'NormalizedResourceTypes: !Join [",", ' + "!Ref InstrumentationResourceTypes]" + ), + 2, + ) + self.assertIn( + 'InstrumentationResourceTypes: !Join [",", ' + "!Ref InstrumentationResourceTypes]", + template, + ) + for resource in ( + "DatadogOperatorSubscriptionLambdaExecutionRole", + "DatadogOperatorSubscriptionFunction", + "DatadogOperatorSubscriptionFunctionTrigger", + ): + self.assertIn(f" {resource}:\n", template) + self.assertEqual(template.count(" Condition: IncludeEKS"), 3) + + role_template = Path(__file__).with_name( + "datadog_integration_role.yaml" + ).read_text() + self.assertIn( + " InstrumentationResourceTypes:\n Type: CommaDelimitedList", + role_template, + ) + self.assertIn( + 'InstrumentationResourceTypes: !Join [",", ' + "!Ref InstrumentationResourceTypes]", + role_template, + ) + + def test_template_grants_only_required_marketplace_actions(self): + template = Path(__file__).with_name( + "datadog_integration_permissions.yaml" + ).read_text() + + actions = ( + "ListPurchaseOptions", + "GetOffer", + "GetOfferTerms", + "SearchAgreements", + "CreateAgreementRequest", + "AcceptAgreementRequest", + "GetAgreementEntitlements", + ) + for action in actions: + self.assertEqual(template.count(f"aws-marketplace:{action}\n"), 1) + self.assertNotIn("aws-marketplace:CancelAgreement", template) + self.assertIn(DATADOG_OPERATOR_PRODUCT_ID, template) + self.assertEqual(template.count(' "Null":'), 3) + + def test_release_embeds_subscription_source(self): + release = Path(__file__).with_name("release.sh").read_text() + + self.assertIn( + "embed_python_source_with_common datadog_integration_permissions.yaml " + "accept_operator_subscription.py ACCEPT_OPERATOR_SUBSCRIPTION_SOURCE", + release, + ) + + +class TestAgreementDiscovery(unittest.TestCase): + def test_returns_active_agreement(self): + client = paginator_client( + search_agreements=[ + {"agreementViewSummaries": [{"agreementId": "agreement-1"}]} + ] + ) + + self.assertEqual(find_active_agreement(client), "agreement-1") + client.paginators["search_agreements"].paginate.assert_called_once_with( + catalog="AWSMarketplace", + filters=[ + {"name": "PartyType", "values": ["Acceptor"]}, + {"name": "AgreementType", "values": ["PurchaseAgreement"]}, + { + "name": "ResourceIdentifier", + "values": [DATADOG_OPERATOR_PRODUCT_ID], + }, + {"name": "Status", "values": ["ACTIVE"]}, + ], + ) + + def test_returns_none_when_no_active_agreement_exists(self): + client = paginator_client( + search_agreements=[{"agreementViewSummaries": []}] + ) + + self.assertIsNone(find_active_agreement(client)) + + def test_rejects_multiple_active_agreements(self): + client = paginator_client( + search_agreements=[ + { + "agreementViewSummaries": [ + {"agreementId": "agreement-1"}, + {"agreementId": "agreement-2"}, + ] + } + ] + ) + + with self.assertRaisesRegex(SubscriptionError, "Multiple active"): + find_active_agreement(client) + + +class TestOfferDiscovery(unittest.TestCase): + def _client(self, options, offers): + client = paginator_client( + list_purchase_options=[{"purchaseOptions": options}] + ) + client.get_offer.side_effect = lambda offerId: offers[offerId] + return client + + def test_selects_only_public_free_offer(self): + client = self._client( + [purchase_option()], + {"offer-1": free_offer()}, + ) + + self.assertEqual(find_free_offer(client)["offerId"], "offer-1") + client.paginators["list_purchase_options"].paginate.assert_called_once_with( + filters=[ + { + "filterType": "PRODUCT_ID", + "filterValues": [DATADOG_OPERATOR_PRODUCT_ID], + }, + { + "filterType": "PURCHASE_OPTION_TYPE", + "filterValues": ["OFFER"], + }, + ] + ) + + def test_rejects_nonfree_and_badged_offers(self): + client = self._client( + [purchase_option("paid"), purchase_option("private")], + { + "paid": free_offer( + "paid", pricingModel={"pricingModelType": "CONTRACT"} + ), + "private": free_offer("private", badges=[{"value": "PRIVATE"}]), + }, + ) + + with self.assertRaisesRegex(SubscriptionError, "No public free"): + find_free_offer(client) + + def test_rejects_multiple_free_offers(self): + client = self._client( + [purchase_option("offer-1"), purchase_option("offer-2")], + { + "offer-1": free_offer("offer-1"), + "offer-2": free_offer("offer-2"), + }, + ) + + with self.assertRaisesRegex(SubscriptionError, "Multiple public free"): + find_free_offer(client) + + def test_skips_unavailable_offer(self): + now = datetime.now(timezone.utc) + client = self._client( + [purchase_option(availableFromTime=now + timedelta(hours=1))], + {"offer-1": free_offer()}, + ) + + with self.assertRaisesRegex(SubscriptionError, "unavailable=1"): + find_free_offer(client, now=now) + client.get_offer.assert_not_called() + + def test_rejects_purchase_option_for_another_product(self): + option = purchase_option() + option["associatedEntities"][0]["product"]["productId"] = "other" + client = self._client([option], {"offer-1": free_offer()}) + + with self.assertRaisesRegex(SubscriptionError, "expected product"): + find_free_offer(client) + + +class TestOfferTerms(unittest.TestCase): + def test_returns_stably_sorted_requested_terms(self): + client = paginator_client( + get_offer_terms=[ + { + "offerTerms": [ + {"supportTerm": {"id": "term-support"}}, + {"legalTerm": {"id": "term-legal"}}, + ] + } + ] + ) + + self.assertEqual( + requested_terms(client, "offer-1"), + [{"id": "term-legal"}, {"id": "term-support"}], + ) + + def test_rejects_missing_or_unsupported_terms(self): + missing = paginator_client( + get_offer_terms=[ + {"offerTerms": [{"legalTerm": {"id": "term-legal"}}]} + ] + ) + unsupported = paginator_client( + get_offer_terms=[ + { + "offerTerms": [ + {"legalTerm": {"id": "term-legal"}}, + {"fixedUpfrontPricingTerm": {"id": "term-price"}}, + ] + } + ] + ) + + with self.assertRaisesRegex(SubscriptionError, "exactly one"): + requested_terms(missing, "offer-1") + with self.assertRaisesRegex(SubscriptionError, "unsupported term"): + requested_terms(unsupported, "offer-1") + + +class TestQuoteValidation(unittest.TestCase): + def test_accepts_only_zero_amounts(self): + validate_zero_charge_summary( + { + "newAgreementValue": "0.00", + "newAgreementValueAfterTax": "0", + "estimatedTaxes": { + "totalAmount": "0", + "breakdown": [{"amount": "0.0"}], + }, + "expectedCharges": [ + { + "amount": "0", + "amountAfterTax": "0", + "estimatedTaxes": {"totalAmount": "0"}, + } + ], + "itemizedCharges": [{"incrementalChargeAmount": "0"}], + } + ) + + def test_rejects_nonzero_and_unknown_amounts(self): + with self.assertRaisesRegex(SubscriptionError, "nonzero charge"): + validate_zero_charge_summary({"newAgreementValue": "0.01"}) + with self.assertRaisesRegex(SubscriptionError, "no charge summary"): + validate_zero_charge_summary(None) + with self.assertRaisesRegex(SubscriptionError, "no value"): + validate_zero_charge_summary({}) + + def test_rejects_after_tax_amounts(self): + with self.assertRaisesRegex(SubscriptionError, "amountAfterTax"): + validate_zero_charge_summary( + { + "newAgreementValue": "0", + "expectedCharges": [ + {"amount": "0", "amountAfterTax": "0.01"} + ], + } + ) + + def test_rejects_future_nested_amount_fields(self): + with self.assertRaisesRegex( + SubscriptionError, + r"futureCharges\[0\]\.serviceFeeAmount", + ): + validate_zero_charge_summary( + { + "newAgreementValue": "0", + "futureCharges": [{"serviceFeeAmount": "1"}], + } + ) + + def test_ignores_nonmonetary_value_fields(self): + validate_zero_charge_summary( + { + "newAgreementValue": "0", + "selectorValue": "paid-plan", + "metadata": {"referenceValue": "1"}, + } + ) + + def test_requires_known_charge_amounts(self): + for summary in ( + {"newAgreementValue": "0", "expectedCharges": [{}]}, + {"newAgreementValue": "0", "itemizedCharges": [{}]}, + ): + with self.subTest(summary=summary): + with self.assertRaisesRegex(SubscriptionError, "no value"): + validate_zero_charge_summary(summary) + + +class TestAgreementCreation(unittest.TestCase): + @patch("accept_operator_subscription.find_free_offer") + @patch("accept_operator_subscription.requested_terms") + def test_creates_validates_and_accepts_agreement(self, mock_terms, mock_offer): + discovery = MagicMock() + agreement = MagicMock() + mock_offer.return_value = free_offer() + mock_terms.return_value = [ + {"id": "term-legal"}, + {"id": "term-support"}, + ] + agreement.create_agreement_request.return_value = { + "agreementRequestId": "request-1", + "chargeSummary": {"newAgreementValue": "0"}, + } + agreement.accept_agreement_request.return_value = { + "agreementId": "agreement-1" + } + + self.assertEqual( + create_and_accept_agreement(event(), discovery, agreement), + "agreement-1", + ) + create_call = agreement.create_agreement_request.call_args.kwargs + self.assertEqual(create_call["agreementProposalIdentifier"], "ap-proposal1") + self.assertEqual(create_call["intent"], "NEW") + self.assertEqual(create_call["requestedTerms"], mock_terms.return_value) + self.assertEqual(len(create_call["clientToken"]), 36) + agreement.accept_agreement_request.assert_called_once_with( + agreementRequestId="request-1" + ) + + @patch("accept_operator_subscription.find_active_agreement") + @patch("accept_operator_subscription.find_free_offer") + @patch("accept_operator_subscription.requested_terms") + def test_recovers_ambiguous_acceptance( + self, + mock_terms, + mock_offer, + mock_find_active, + ): + mock_offer.return_value = free_offer() + mock_terms.return_value = [{"id": "legal"}, {"id": "support"}] + mock_find_active.return_value = "agreement-1" + agreement = MagicMock() + agreement.create_agreement_request.return_value = { + "agreementRequestId": "request-1", + "chargeSummary": {"newAgreementValue": "0"}, + } + agreement.accept_agreement_request.side_effect = TimeoutError("timed out") + + self.assertEqual( + create_and_accept_agreement(event(), MagicMock(), agreement), + "agreement-1", + ) + + @patch("accept_operator_subscription.time.monotonic", side_effect=[0, 100]) + @patch("accept_operator_subscription.find_active_agreement") + @patch("accept_operator_subscription.find_free_offer") + @patch("accept_operator_subscription.requested_terms") + def test_does_not_recover_when_acceptance_was_not_attempted( + self, + mock_terms, + mock_offer, + mock_find_active, + _mock_monotonic, + ): + mock_offer.return_value = free_offer() + mock_terms.return_value = [{"id": "legal"}, {"id": "support"}] + agreement = MagicMock() + agreement.create_agreement_request.return_value = { + "agreementRequestId": "request-1", + "chargeSummary": {"newAgreementValue": "0"}, + } + + with self.assertRaisesRegex(RuntimeError, "accept_agreement_request"): + create_and_accept_agreement( + event(), + MagicMock(), + agreement, + deadline=120, + ) + + agreement.accept_agreement_request.assert_not_called() + mock_find_active.assert_not_called() + + +class TestEntitlements(unittest.TestCase): + def test_returns_only_operator_entitlement_across_pages(self): + client = paginator_client( + get_agreement_entitlements=[ + { + "agreementEntitlements": [ + {"resource": {"id": "other"}, "status": "PROVISIONED"} + ] + }, + { + "agreementEntitlements": [ + { + "resource": {"id": DATADOG_OPERATOR_PRODUCT_ID}, + "status": "PENDING", + } + ] + }, + ] + ) + + self.assertEqual( + entitlement_status(client, "agreement-1")["status"], "PENDING" + ) + paginator = client.paginators["get_agreement_entitlements"] + paginator.paginate.assert_called_once_with(agreementId="agreement-1") + + @patch("accept_operator_subscription.time.sleep") + @patch("accept_operator_subscription.entitlement_status") + def test_waits_until_provisioned(self, mock_status, mock_sleep): + mock_status.side_effect = [ + {"status": "PENDING", "statusReasonCode": "PROVISIONING_IN_PROGRESS"}, + {"status": "PROVISIONED", "statusReasonCode": "AGREEMENT_ACTIVE"}, + ] + + wait_for_entitlement(MagicMock(), "agreement-1", attempts=2, delay=1) + + mock_sleep.assert_called_once_with(1) + + def test_fails_terminal_status_and_timeout(self): + with patch( + "accept_operator_subscription.entitlement_status", + return_value={"status": "FAILED", "statusReasonCode": "PRODUCT_RESTRICTED"}, + ): + with self.assertRaisesRegex(SubscriptionError, "FAILED"): + wait_for_entitlement(MagicMock(), "agreement-1", attempts=1) + with patch( + "accept_operator_subscription.entitlement_status", return_value=None + ): + with self.assertRaisesRegex(SubscriptionError, "Timed out"): + wait_for_entitlement(MagicMock(), "agreement-1", attempts=1) + + @patch("accept_operator_subscription.time.monotonic", return_value=70) + def test_stops_before_deadline_without_starting_request(self, _mock_monotonic): + client = MagicMock() + + with self.assertRaisesRegex(SubscriptionError, "Timed out"): + wait_for_entitlement(client, "agreement-1", deadline=100) + + client.get_paginator.assert_not_called() + + @patch("accept_operator_subscription.time.sleep") + @patch("accept_operator_subscription.entitlement_status") + @patch("accept_operator_subscription.time.monotonic", side_effect=[0, 65]) + def test_preserves_last_status_when_next_request_exceeds_budget( + self, + _mock_monotonic, + mock_status, + mock_sleep, + ): + mock_status.return_value = { + "status": "PENDING", + "statusReasonCode": "PROVISIONING_IN_PROGRESS", + } + + with self.assertRaises(SubscriptionError) as raised: + wait_for_entitlement( + MagicMock(), + "agreement-1", + attempts=2, + delay=5, + deadline=100, + ) + + self.assertEqual(raised.exception.reason, "entitlement_timeout") + self.assertIn("status=PENDING", str(raised.exception)) + mock_status.assert_called_once() + mock_sleep.assert_not_called() + + +class TestAPIFailureStages(unittest.TestCase): + def assert_stage(self, expected_stage, operation): + with self.assertRaises(SubscriptionError) as raised: + operation() + self.assertEqual(raised.exception.stage, expected_stage) + self.assertEqual(raised.exception.reason, "aws_api_error") + self.assertIn("AccessDenied", str(raised.exception)) + + def test_agreement_discovery_failure(self): + client = MagicMock() + client.get_paginator.side_effect = RuntimeError("AccessDenied") + + self.assert_stage( + "agreement_discovery", lambda: find_active_agreement(client) + ) + + @patch("accept_operator_subscription.time.monotonic", return_value=90) + def test_agreement_discovery_stops_near_deadline(self, _mock_monotonic): + client = paginator_client( + search_agreements=[{"agreementViewSummaries": []}] + ) + + with self.assertRaisesRegex(SubscriptionError, "deadline is near"): + find_active_agreement(client, deadline=100) + + def test_offer_discovery_failure(self): + client = MagicMock() + client.get_paginator.side_effect = RuntimeError("AccessDenied") + + self.assert_stage("offer_discovery", lambda: find_free_offer(client)) + + def test_offer_terms_failure(self): + client = MagicMock() + client.get_paginator.side_effect = RuntimeError("AccessDenied") + + self.assert_stage( + "offer_terms", lambda: requested_terms(client, "offer-1") + ) + + def test_request_creation_failure(self): + agreement = MagicMock() + agreement.create_agreement_request.side_effect = RuntimeError("AccessDenied") + with ( + patch( + "accept_operator_subscription.find_free_offer", + return_value=free_offer(), + ), + patch( + "accept_operator_subscription.requested_terms", + return_value=[{"id": "legal"}, {"id": "support"}], + ), + ): + self.assert_stage( + "request_creation", + lambda: create_and_accept_agreement( + event(), MagicMock(), agreement + ), + ) + + def test_request_acceptance_failure(self): + agreement = MagicMock() + agreement.create_agreement_request.return_value = { + "agreementRequestId": "request-1", + "chargeSummary": {"newAgreementValue": "0"}, + } + agreement.accept_agreement_request.side_effect = RuntimeError("AccessDenied") + with ( + patch( + "accept_operator_subscription.find_free_offer", + return_value=free_offer(), + ), + patch( + "accept_operator_subscription.requested_terms", + return_value=[{"id": "legal"}, {"id": "support"}], + ), + patch( + "accept_operator_subscription.find_active_agreement", + return_value=None, + ), + ): + self.assert_stage( + "request_acceptance", + lambda: create_and_accept_agreement( + event(), MagicMock(), agreement + ), + ) + + +class TestClientInitialization(unittest.TestCase): + @patch("accept_operator_subscription.boto3.Session") + def test_reports_runtime_without_marketplace_discovery(self, mock_session): + mock_session.return_value.client.side_effect = RuntimeError("UnknownServiceError") + + with self.assertRaises(SubscriptionError) as raised: + ensure_subscription(event()) + + self.assertEqual(raised.exception.stage, "sdk_initialization") + self.assertEqual(raised.exception.reason, "unsupported_sdk") + self.assertIn("UnknownServiceError", str(raised.exception)) + + +class TestHandler(unittest.TestCase): + def setUp(self): + self.context = MagicMock() + self.context.get_remaining_time_in_millis.return_value = 300_000 + sys.modules["cfnresponse"].send.reset_mock() + + def response(self): + return sys.modules["cfnresponse"].send.call_args + + @patch("accept_operator_subscription.boto3.Session") + def test_delete_retains_agreement_without_aws_calls(self, mock_session): + handler(event(request_type="Delete"), self.context) + + mock_session.assert_not_called() + self.assertEqual(self.response().args[2], "SUCCESS") + self.assertEqual( + self.response().kwargs["responseData"], {"AgreementRetained": True} + ) + + @patch("accept_operator_subscription.time.monotonic", return_value=100) + @patch("accept_operator_subscription.ensure_subscription") + def test_returns_agreement_details(self, mock_ensure, _mock_monotonic): + mock_ensure.return_value = "agreement-1" + + handler(event(), self.context) + + self.assertEqual(self.response().args[2], "SUCCESS") + self.assertEqual( + self.response().kwargs["responseData"], + {"AgreementId": "agreement-1"}, + ) + mock_ensure.assert_called_once_with(event(), deadline=385) + + @patch("accept_operator_subscription.ensure_subscription") + def test_reports_actionable_failure(self, mock_ensure): + mock_ensure.side_effect = SubscriptionError( + "offer_discovery", "no_eligible_free_offer", "No free offer" + ) + + handler(event(), self.context) + + self.assertEqual(self.response().args[2], "FAILED") + self.assertEqual( + self.response().kwargs["responseData"], {"Message": "No free offer"} + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 784551fe..39be6110 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -10,6 +10,8 @@ import cfnresponse import boto3 +from cfn_common import send_cfn_response + LOGGER = logging.getLogger() LOGGER.setLevel(logging.INFO) API_CALL_SOURCE_HEADER_VALUE = "cfn-quickstart" @@ -142,20 +144,6 @@ def _should_update_instrumentation_permissions(event): ) -def _physical_resource_id(event): - return event.get("PhysicalResourceId") or f"{event['StackId']}/{event['LogicalResourceId']}" - - -def _send_cfn_response(event, context, status, response_data): - cfnresponse.send( - event, - context, - status, - responseData=response_data, - physicalResourceId=_physical_resource_id(event), - ) - - def build_instrumentation_permissions_url(datadog_site, resource_types, account_id, partition): query = urllib.parse.urlencode( [("resource_type", t) for t in resource_types] @@ -1209,10 +1197,14 @@ def handle_delete(event, context): response_data = {} if preserved_boundaries: response_data["PreservedPermissionsBoundaries"] = preserved_boundaries - _send_cfn_response(event, context, cfnresponse.SUCCESS, response_data) + send_cfn_response( + cfnresponse, event, context, cfnresponse.SUCCESS, response_data + ) except Exception as e: LOGGER.error(f"Error deleting policy: {str(e)}") - _send_cfn_response(event, context, cfnresponse.FAILED, {"Message": str(e)}) + send_cfn_response( + cfnresponse, event, context, cfnresponse.FAILED, {"Message": str(e)} + ) def handle_create_update(event, context): @@ -1263,10 +1255,12 @@ def handle_create_update(event, context): ) if target_changed: _cleanup_previous_target_policies(iam_client, previous_props) - _send_cfn_response(event, context, cfnresponse.SUCCESS, {}) + send_cfn_response(cfnresponse, event, context, cfnresponse.SUCCESS, {}) except Exception as e: LOGGER.error(f"Error creating/attaching policy: {str(e)}") - _send_cfn_response(event, context, cfnresponse.FAILED, {"Message": str(e)}) + send_cfn_response( + cfnresponse, event, context, cfnresponse.FAILED, {"Message": str(e)} + ) def handler(event, context): diff --git a/aws_quickstart/attach_integration_permissions_test.py b/aws_quickstart/attach_integration_permissions_test.py index 50a073b7..35d6fca0 100644 --- a/aws_quickstart/attach_integration_permissions_test.py +++ b/aws_quickstart/attach_integration_permissions_test.py @@ -60,21 +60,21 @@ def test_cloudformation_template_uses_source_placeholder(self): template = template_path.read_text() self.assertIn( - " Code:\n ZipFile: |\n \n", + " Code:\n ZipFile: |\n \n", template, ) - self.assertEqual(template.count(""), 1) + self.assertEqual(template.count(""), 1) def test_release_embeds_tested_source(self): release_path = Path(__file__).with_name("release.sh") release = release_path.read_text() self.assertIn( - 'cp datadog_agentless_api_call.py attach_integration_permissions.py "${TEMP_DIR}/"', + 'cp datadog_agentless_api_call.py cfn_common.py attach_integration_permissions.py accept_operator_subscription.py "${TEMP_DIR}/"', release, ) self.assertIn( - "embed_python_source datadog_integration_permissions.yaml attach_integration_permissions.py", + "embed_python_source_with_common datadog_integration_permissions.yaml attach_integration_permissions.py ATTACH_INTEGRATION_PERMISSIONS_SOURCE", release, ) diff --git a/aws_quickstart/cfn_common.py b/aws_quickstart/cfn_common.py new file mode 100644 index 00000000..0fb597a1 --- /dev/null +++ b/aws_quickstart/cfn_common.py @@ -0,0 +1,14 @@ +def physical_resource_id(event): + return event.get("PhysicalResourceId") or ( + f"{event['StackId']}/{event['LogicalResourceId']}" + ) + + +def send_cfn_response(cfn_response, event, context, status, response_data): + cfn_response.send( + event, + context, + status, + responseData=response_data, + physicalResourceId=physical_resource_id(event), + ) diff --git a/aws_quickstart/cfn_common_test.py b/aws_quickstart/cfn_common_test.py new file mode 100644 index 00000000..c70afb08 --- /dev/null +++ b/aws_quickstart/cfn_common_test.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import unittest +from unittest.mock import MagicMock + +from cfn_common import physical_resource_id, send_cfn_response + + +def event(**overrides): + value = { + "StackId": "arn:aws:cloudformation:us-east-1:123456789012:stack/test/id", + "LogicalResourceId": "CustomResource", + } + value.update(overrides) + return value + + +class TestPhysicalResourceId(unittest.TestCase): + def test_preserves_existing_id(self): + self.assertEqual( + physical_resource_id(event(PhysicalResourceId="existing-id")), + "existing-id", + ) + + def test_builds_deterministic_id(self): + value = event() + self.assertEqual( + physical_resource_id(value), + f"{value['StackId']}/{value['LogicalResourceId']}", + ) + + +class TestSendCfnResponse(unittest.TestCase): + def test_sends_response_with_physical_resource_id(self): + cfn_response = MagicMock() + value = event() + + send_cfn_response( + cfn_response, + value, + "context", + "SUCCESS", + {"Result": "ok"}, + ) + + cfn_response.send.assert_called_once_with( + value, + "context", + "SUCCESS", + responseData={"Result": "ok"}, + physicalResourceId=f"{value['StackId']}/{value['LogicalResourceId']}", + ) + + +class TestInlineComposition(unittest.TestCase): + def test_shared_helper_composes_with_each_handler(self): + directory = Path(__file__).parent + common = (directory / "cfn_common.py").read_text() + + for filename in ( + "attach_integration_permissions.py", + "accept_operator_subscription.py", + ): + handler = (directory / filename).read_text().replace( + "from cfn_common import send_cfn_response\n", "" + ) + source = f"{common}\n{handler}" + + with self.subTest(filename=filename): + self.assertNotIn("from cfn_common import", source) + compile(source, filename, "exec") + + +if __name__ == "__main__": + unittest.main() diff --git a/aws_quickstart/datadog_integration_permissions.yaml b/aws_quickstart/datadog_integration_permissions.yaml index 16b1ae43..45792f71 100644 --- a/aws_quickstart/datadog_integration_permissions.yaml +++ b/aws_quickstart/datadog_integration_permissions.yaml @@ -14,12 +14,13 @@ Parameters: Description: >- Set this value to "true" to add permissions for Datadog to collect resource configuration data. InstrumentationResourceTypes: - Type: String + Type: CommaDelimitedList Default: "" Description: >- Comma-separated list of AWS resource types (UDM form, e.g. aws:ec2:instance, aws:ecs:cluster, aws:eks:cluster) that the Datadog integration role should be granted the IAM permissions - required to instrument with the Datadog Agent. Leave blank to skip. + required to instrument with the Datadog Agent. In the commercial AWS partition, selecting + EKS also accepts the free Datadog Operator AWS Marketplace agreement. Leave blank to skip. DatadogSite: Type: String Default: "datadoghq.com" @@ -48,6 +49,24 @@ Parameters: an optional add-on to the broader install. The post-setup add-on sets this to "true" because attaching the instrumentation permissions is the stack's only purpose. Updates that replace existing instrumentation permissions still fail atomically so stale policies are not accepted. +Conditions: + IncludeEKS: + Fn::And: + - Fn::Equals: + - !Ref AWS::Partition + - aws + - Fn::Not: + - Fn::Equals: + - !Join + - "" + - !Split + - ",aws:eks:cluster," + - !Sub + - ",${NormalizedResourceTypes}," + - NormalizedResourceTypes: !Join [",", !Ref InstrumentationResourceTypes] + - !Sub + - ",${NormalizedResourceTypes}," + - NormalizedResourceTypes: !Join [",", !Ref InstrumentationResourceTypes] Resources: DatadogAttachIntegrationPermissionsLambdaExecutionRole: Type: AWS::IAM::Role @@ -133,7 +152,7 @@ Resources: Timeout: 300 Code: ZipFile: | - + DatadogAttachIntegrationPermissionsFunctionTrigger: Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger Properties: @@ -144,7 +163,93 @@ Resources: AccountId: !Ref AWS::AccountId Partition: !Sub "${AWS::Partition}" ResourceCollectionPermissions: !Ref ResourceCollectionPermissions - InstrumentationResourceTypes: !Ref InstrumentationResourceTypes + InstrumentationResourceTypes: !Join [",", !Ref InstrumentationResourceTypes] DatadogSite: !Ref DatadogSite ManageBasePermissions: !Ref ManageBasePermissions FailOnInstrumentationError: !Ref FailOnInstrumentationError + DatadogOperatorSubscriptionLambdaExecutionRole: + Type: AWS::IAM::Role + Condition: IncludeEKS + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + Policies: + - PolicyName: datadog-operator-marketplace-subscription + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: aws-marketplace:ListPurchaseOptions + Resource: !Sub arn:${AWS::Partition}:aws-marketplace:::catalog/AWSMarketplace/purchaseOption/* + - Effect: Allow + Action: + - aws-marketplace:GetOffer + - aws-marketplace:GetOfferTerms + Resource: !Sub arn:${AWS::Partition}:aws-marketplace:::catalog/AWSMarketplace/offer/* + - Effect: Allow + Action: + - aws-marketplace:CreateAgreementRequest + - aws-marketplace:AcceptAgreementRequest + Resource: "*" + Condition: + "Null": + aws-marketplace:AgreementType: "false" + aws-marketplace:ProductId: "false" + ForAllValues:StringEquals: + aws-marketplace:AgreementType: + - PurchaseAgreement + ForAnyValue:StringEquals: + aws-marketplace:ProductId: + - 6e852b2a-ecbb-431c-9b63-7de0288f4d00 + - Effect: Allow + Action: aws-marketplace:SearchAgreements + Resource: "*" + Condition: + "Null": + aws-marketplace:AgreementType: "false" + ForAllValues:StringEquals: + aws-marketplace:AgreementType: + - PurchaseAgreement + StringEquals: + aws-marketplace:PartyType: Acceptor + - Effect: Allow + Action: aws-marketplace:GetAgreementEntitlements + Resource: "*" + Condition: + "Null": + aws-marketplace:AgreementType: "false" + ForAllValues:StringEquals: + aws-marketplace:AgreementType: + - PurchaseAgreement + DatadogOperatorSubscriptionFunction: + Type: AWS::Lambda::Function + Condition: IncludeEKS + Properties: + Description: Accepts the free Datadog Operator AWS Marketplace agreement. + Role: !GetAtt DatadogOperatorSubscriptionLambdaExecutionRole.Arn + Handler: index.handler + LoggingConfig: + ApplicationLogLevel: INFO + LogFormat: JSON + Runtime: python3.14 + Timeout: 300 + Code: + ZipFile: | + + DatadogOperatorSubscriptionFunctionTrigger: + Type: Custom::DatadogOperatorSubscriptionFunctionTrigger + Condition: IncludeEKS + Properties: + ServiceToken: !GetAtt DatadogOperatorSubscriptionFunction.Arn + SubscriptionSchemaVersion: "1" + AccountId: !Ref AWS::AccountId diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 0a3cf14e..f3285ed0 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -19,7 +19,7 @@ Parameters: Description: >- Set this value to "true" to add permissions for Datadog to collect resource configuration data. InstrumentationResourceTypes: - Type: String + Type: CommaDelimitedList Default: "" Description: >- Comma-separated list of AWS resource types (UDM form, e.g. aws:ec2:instance, aws:ecs:cluster, @@ -85,7 +85,7 @@ Resources: Parameters: IAMRoleName: !Ref IAMRoleName ResourceCollectionPermissions: !Ref ResourceCollectionPermissions - InstrumentationResourceTypes: !Ref InstrumentationResourceTypes + InstrumentationResourceTypes: !Join [",", !Ref InstrumentationResourceTypes] DatadogSite: !Ref DatadogSite ManageBasePermissions: true Metadata: diff --git a/aws_quickstart/release.sh b/aws_quickstart/release.sh index 5456749c..035358e0 100755 --- a/aws_quickstart/release.sh +++ b/aws_quickstart/release.sh @@ -44,15 +44,36 @@ upload_versions_json() { embed_python_source() { local template="$1" local source="$2" + local placeholder="${3:-ZIPFILE_PLACEHOLDER}" - perl -i -pe ' - BEGIN { $p = do { local $/; } } - /^(\s+)/ && ( + DD_QUICKSTART_SOURCE_PLACEHOLDER="<${placeholder}>" perl -i -pe ' + BEGIN { + $p = do { local $/; }; + $placeholder = $ENV{"DD_QUICKSTART_SOURCE_PLACEHOLDER"}; + } + /^(\s+)\Q$placeholder\E\s*$/ && ( $_ = join("\n", map { $1 . $_ } split(/\n/, $p)) . "\n" ) ' "$template" < "$source" } +embed_python_source_with_common() { + local template="$1" + local source="$2" + local placeholder="$3" + local composed_source + composed_source=$(mktemp) + + # CloudFormation ZipFile Lambdas are single-file modules, so inline the shared + # helper and remove the local-development import from the handler source. + { + sed -e '$a\' cfn_common.py + sed '/^from cfn_common import send_cfn_response$/d' "$source" + } > "${composed_source}" + embed_python_source "$template" "${composed_source}" "$placeholder" + rm -f "${composed_source}" +} + # Parse flags and optional bucket argument GOV=false PRIVATE_TEMPLATE=false @@ -134,7 +155,7 @@ trap "rm -rf ${TEMP_DIR}" EXIT # Copy all YAML files to temp directory cp *.yaml "${TEMP_DIR}/" -cp datadog_agentless_api_call.py attach_integration_permissions.py "${TEMP_DIR}/" +cp datadog_agentless_api_call.py cfn_common.py attach_integration_permissions.py accept_operator_subscription.py "${TEMP_DIR}/" # Change to temp directory for processing cd "${TEMP_DIR}" @@ -161,7 +182,8 @@ for template in main_workflow.yaml main_extended_workflow.yaml main_v2.yaml main fi done -embed_python_source datadog_integration_permissions.yaml attach_integration_permissions.py +embed_python_source_with_common datadog_integration_permissions.yaml attach_integration_permissions.py ATTACH_INTEGRATION_PERMISSIONS_SOURCE +embed_python_source_with_common datadog_integration_permissions.yaml accept_operator_subscription.py ACCEPT_OPERATOR_SUBSCRIPTION_SOURCE # Process Agentless Scanning templates for template in datadog_agentless_delegate_role.yaml datadog_agentless_scanning.yaml datadog_agentless_delegate_role_snapshot.yaml datadog_integration_autoscaling_policy.yaml datadog_integration_sds_policy.yaml datadog_agentless_delegate_role_stackset.yaml datadog_agentless_saas.yaml; do diff --git a/aws_quickstart/version.txt b/aws_quickstart/version.txt index 347b96f7..b4abc501 100644 --- a/aws_quickstart/version.txt +++ b/aws_quickstart/version.txt @@ -1 +1 @@ -v4.18.1 +v4.19.0