diff --git a/aws_cloud_cost_cur2/CHANGELOG.md b/aws_cloud_cost_cur2/CHANGELOG.md deleted file mode 100644 index 46cb5656..00000000 --- a/aws_cloud_cost_cur2/CHANGELOG.md +++ /dev/null @@ -1,19 +0,0 @@ -## 0.0.6 - -Fix IAM policy creation by replacing `/` with `_` in the report prefix when building the policy name. - -## 0.0.5 - -Wait for CUR export to finish creating before calling Datadog API. - -## 0.0.4 - -Add `uk1.datadoghq.com` to the `DatadogSite` allowed values (UK1 site support). - -## 0.0.3 - -Add `us2.ddog-gov.com` to the `DatadogSite` allowed values (us2.fed GovCloud support). - -## 0.0.1 - -Initial version diff --git a/aws_cloud_cost_cur2/README.md b/aws_cloud_cost_cur2/README.md deleted file mode 100644 index 0609a824..00000000 --- a/aws_cloud_cost_cur2/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Datadog AWS Cloud Cost Setup (CUR 2.0) - -[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](ADDLINK) - -## Overview - -This template sets up AWS Cloud Cost Management for Datadog using **CUR 2.0 (Data Exports)**. It creates the necessary AWS resources and configures the Datadog integration automatically. - -## AWS Resources - -This template creates the following: - -- **S3 Bucket** (optional) - For storing cost and usage data - - Bucket policy allowing `bcm-data-exports.amazonaws.com` to write reports -- **BCM Data Export** (optional) - CUR 2.0 export with hourly granularity in Parquet format -- **IAM Policy** - Grants Datadog read access to the S3 bucket and Cost Explorer APIs - - Attached to your existing Datadog integration role -- **Lambda Function** - Calls the Datadog API to configure CCM with the data export details - -## Parameters - -| Parameter | Description | Required | -|-----------|-------------|----------| -| `DatadogApiKey` | Datadog API key | Yes | -| `DatadogAppKey` | Datadog APP key | Yes | -| `DatadogSite` | Datadog site (e.g., datadoghq.com) | No (default: datadoghq.com) | -| `CloudCostBucketName` | S3 bucket name for cost reports | Yes | -| `CloudCostBucketRegion` | AWS region of the S3 bucket | Yes | -| `CloudCostReportPrefix` | Prefix for report files in S3 | Yes | -| `CloudCostReportName` | Name of the data export | Yes | -| `DatadogIntegrationRole` | Name of your Datadog integration IAM role | Yes | -| `CreateCloudCostReport` | Create a new data export (true/false) | No (default: true) | -| `CreateCloudCostBucket` | Create a new S3 bucket (true/false) | No (default: true) | - -## Publishing the Template - -Use the release script to upload the template to an S3 bucket: - -```sh -./release.sh -``` - -Use `--private` to prevent granting public access (useful for testing): - -```sh -./release.sh --private -``` - -The template will be available at `s3:///aws_cloud_cost_cur2/v0.0.2/main.yaml`. diff --git a/aws_cloud_cost_cur2/datadog_ccm_api_call.py b/aws_cloud_cost_cur2/datadog_ccm_api_call.py deleted file mode 100644 index c81511c4..00000000 --- a/aws_cloud_cost_cur2/datadog_ccm_api_call.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 - -import json -import logging -import urllib.request -import urllib.error -import cfnresponse - -LOGGER = logging.getLogger() -LOGGER.setLevel(logging.INFO) - -API_CALL_SOURCE_HEADER_VALUE = "cfn-ccm-cur2" - - -def get_datadog_account_uuid(event): - """Get the Datadog account UUID for this AWS account.""" - api_key = event["ResourceProperties"]["APIKey"] - app_key = event["ResourceProperties"]["APPKey"] - api_url = event["ResourceProperties"]["ApiURL"] - account_id = event["ResourceProperties"]["AccountId"] - - url = f"https://api.{api_url}/api/v2/integration/aws/accounts?aws_account_id={account_id}" - headers = { - "DD-API-KEY": api_key, - "DD-APPLICATION-KEY": app_key, - "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, - } - request = urllib.request.Request(url, headers=headers) - request.get_method = lambda: "GET" - try: - response = urllib.request.urlopen(request) - data = json.loads(response.read()) - if len(data.get("data", [])) == 0: - return None, "No Datadog integration found for this AWS account" - if len(data["data"]) > 1: - return None, "Multiple Datadog integrations found for this AWS account" - return data["data"][0]["id"], None - except urllib.error.HTTPError as e: - error_body = e.read().decode("utf-8") - return None, f"Failed to get account: {e.code} - {error_body}" - - -def create_ccm_config(event, uuid): - """Create CCM config using the dedicated CCM config endpoint.""" - api_key = event["ResourceProperties"]["APIKey"] - app_key = event["ResourceProperties"]["APPKey"] - api_url = event["ResourceProperties"]["ApiURL"] - bucket_name = event["ResourceProperties"]["BucketName"] - bucket_region = event["ResourceProperties"]["BucketRegion"] - report_name = event["ResourceProperties"]["ReportName"] - report_prefix = event["ResourceProperties"]["ReportPrefix"] - - url = f"https://api.{api_url}/api/v2/integration/aws/accounts/{uuid}/ccm_config" - headers = { - "DD-API-KEY": api_key, - "DD-APPLICATION-KEY": app_key, - "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, - "Content-Type": "application/json", - "Accept": "application/json", - } - - payload = { - "data": { - "type": "account", - "attributes": { - "ccm_config": { - "data_export_configs": [ - { - "report_name": report_name, - "report_prefix": report_prefix, - "report_type": "CUR2.0", - "bucket_name": bucket_name, - "bucket_region": bucket_region, - } - ] - } - } - } - } - - data = json.dumps(payload).encode("utf-8") - request = urllib.request.Request(url, data=data, headers=headers, method="POST") - - try: - response = urllib.request.urlopen(request) - return response.getcode(), json.loads(response.read()) - except urllib.error.HTTPError as e: - error_body = e.read().decode("utf-8") - return e.code, {"error": error_body} - - -def handler(event, context): - """Handle Lambda event from AWS CloudFormation.""" - LOGGER.info(f"Received event: {json.dumps(event)}") - - if event["RequestType"] == "Delete": - # On delete, we don't remove the CCM config - just succeed - LOGGER.info("Delete request - no action needed for CCM config") - cfnresponse.send(event, context, cfnresponse.SUCCESS, {"Message": "Delete successful"}) - return - - try: - # Get the Datadog account UUID - uuid, error = get_datadog_account_uuid(event) - if error: - LOGGER.error(f"Failed to get account UUID: {error}") - cfnresponse.send(event, context, cfnresponse.FAILED, {"Message": error}) - return - - LOGGER.info(f"Found Datadog account UUID: {uuid}") - - # Create the CCM config using the dedicated endpoint - status_code, response_data = create_ccm_config(event, uuid) - - if status_code == 200: - LOGGER.info("Successfully configured CCM data export") - cfnresponse.send(event, context, cfnresponse.SUCCESS, { - "Message": "CCM data export configured successfully", - "AccountUUID": uuid, - }) - else: - error_msg = f"API returned {status_code}: {response_data}" - LOGGER.error(error_msg) - cfnresponse.send(event, context, cfnresponse.FAILED, {"Message": error_msg}) - - except Exception as e: - LOGGER.exception("Exception during processing") - cfnresponse.send(event, context, cfnresponse.FAILED, {"Message": str(e)}) - diff --git a/aws_cloud_cost_cur2/main.yaml b/aws_cloud_cost_cur2/main.yaml deleted file mode 100644 index b0445e09..00000000 --- a/aws_cloud_cost_cur2/main.yaml +++ /dev/null @@ -1,330 +0,0 @@ -# version: 0.0.6 -AWSTemplateFormatVersion: 2010-09-09 -Description: Datadog Cloud Cost Management Setup -Parameters: - DatadogApiKey: - Type: String - Description: API key for the Datadog account. - NoEcho: true - MinLength: 1 - DatadogAppKey: - Type: String - Description: APP key for the Datadog account. - NoEcho: true - MinLength: 1 - DatadogSite: - Type: String - Default: datadoghq.com - Description: Define your Datadog Site to send data to. - AllowedValues: - - datadoghq.com - - datadoghq.eu - - us3.datadoghq.com - - us5.datadoghq.com - - ap1.datadoghq.com - - ap2.datadoghq.com - - ddog-gov.com - - us2.ddog-gov.com - - uk1.datadoghq.com - CloudCostBucketName: - Type: String - Description: The name of the S3 bucket for storing the cost and usage reports. - MinLength: 1 - CloudCostBucketRegion: - Type: String - Description: The AWS region where the S3 bucket is located. - MinLength: 1 - CloudCostReportPrefix: - Type: String - Description: The prefix to be added to the cost and usage report files in the S3 bucket. - MinLength: 1 - CloudCostReportName: - Type: String - Description: The name of the cost and usage report. - MinLength: 1 - DatadogIntegrationRole: - Type: String - Description: The name of the IAM role that has been integrated with Datadog. - MinLength: 1 - CreateCloudCostReport: - Type: String - Default: true - AllowedValues: - - true - - false - Description: Whether a new Cost and Usage Report is created, or an existing one is used. - CreateCloudCostBucket: - Type: String - Default: true - AllowedValues: - - true - - false - Description: Whether the S3 bucket for storing the Cost and Usage Report is created, or an existing one is used. Ignored if CreateCloudCostReport is false. - -Conditions: - ShouldCreateCloudCostReport: - Fn::Equals: - - Ref: CreateCloudCostReport - - true - ShouldCreateCloudCostBucket: - Fn::And: - - Fn::Equals: - - Ref: CreateCloudCostBucket - - true - - !Condition ShouldCreateCloudCostReport # If CUR already exists, we don't need to create the bucket - -Resources: - DatadogCCMIAMPolicy: - Type: AWS::IAM::RolePolicy - Properties: - PolicyName: - Fn::Sub: - - "datadog-ccm-iam-policy--${CloudCostBucketName}-${SanitizedCloudCostReportPrefix}-${CloudCostReportName}" - - SanitizedCloudCostReportPrefix: - Fn::Join: - - "_" - - Fn::Split: - - "/" - - Ref: CloudCostReportPrefix - RoleName: !Ref DatadogIntegrationRole - PolicyDocument: - Version: "2012-10-17" - Statement: - - Sid: ReadCloudCostBucket - Effect: Allow - Action: - - s3:ListBucket - Resource: !Sub "arn:${AWS::Partition}:s3:::${CloudCostBucketName}" - - Sid: GetBill - Effect: Allow - Action: - - s3:GetObject - Resource: !Sub "arn:${AWS::Partition}:s3:::${CloudCostBucketName}/${CloudCostReportPrefix}/${CloudCostReportName}/*" - - Sid: CheckAccuracy - Effect: Allow - Action: - - ce:Get* - Resource: "*" - - Sid: ListCURs - Effect: Allow - Action: - - cur:DescribeReportDefinitions - Resource: "*" - - Sid: ListOrganizations - Effect: Allow - Action: - - organizations:Describe* - - organizations:List* - Resource: "*" - - DDCloudCostBucket: - Condition: ShouldCreateCloudCostBucket - Type: AWS::S3::Bucket - Properties: - BucketName: !Ref CloudCostBucketName - - DDCloudCostBucketAccessPolicy: - Condition: ShouldCreateCloudCostBucket - Type: AWS::S3::BucketPolicy - Properties: - Bucket: !Ref DDCloudCostBucket - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: - - billingreports.amazonaws.com - - bcm-data-exports.amazonaws.com - Action: - - s3:PutObject - - s3:GetBucketPolicy - Resource: - - !Sub ${DDCloudCostBucket.Arn} - - !Sub ${DDCloudCostBucket.Arn}/* - Condition: - StringLike: - aws:SourceArn: - - !Sub arn:*:cur:*:${AWS::AccountId}:definition/* - - !Sub arn:*:bcm-data-exports:*:${AWS::AccountId}:export/* - aws:SourceAccount: !Sub ${AWS::AccountId} - - # DDCURReportDefinition can only be done after the bucket is created - # and has the right policies. The bucket is only created if ShouldCreateCloudCostBucket - # is true. DependsOn does not work with Conditions, so we use a WaitCondition - # There are 2 wait handles, - # - BucketWaitHandle: for the bucket if bucket should be created - # - EmptyWaitHandle: for empty wait condition if bucket should not be created - # BucketWaitCondition to join them and be used as a dependency for DDCURReportDefinition - # More info here - https://garbe.io/blog/2017/07/17/cloudformation-hacks/ - BucketWaitHandle: - Condition: ShouldCreateCloudCostBucket - DependsOn: DDCloudCostBucketAccessPolicy - Type: "AWS::CloudFormation::WaitConditionHandle" - - EmptyWaitHandle: - Type: "AWS::CloudFormation::WaitConditionHandle" - - BucketWaitCondition: - Type: "AWS::CloudFormation::WaitCondition" - Properties: - Handle: - !If [ - ShouldCreateCloudCostBucket, - !Ref BucketWaitHandle, - !Ref EmptyWaitHandle, - ] - Timeout: "1" - Count: 0 - - DDCURReportDefinition: - Condition: ShouldCreateCloudCostReport - DependsOn: BucketWaitCondition - Type: AWS::BCMDataExports::Export - Properties: - Export: - DataQuery: - QueryStatement: > - SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, - bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, - bill_payer_account_id, bill_payer_account_name, cost_category, discount, - discount_bundled_discount, discount_total_discount, identity_line_item_id, - identity_time_interval, line_item_availability_zone, line_item_blended_cost, - line_item_blended_rate, line_item_currency_code, line_item_legal_entity, - line_item_line_item_description, line_item_line_item_type, - line_item_net_unblended_cost, line_item_net_unblended_rate, - line_item_normalization_factor, line_item_normalized_usage_amount, - line_item_operation, line_item_product_code, line_item_resource_id, - line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, - line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, - line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, - pricing_currency, pricing_lease_contract_length, pricing_offering_class, - pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, - pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, - product_comment, product_fee_code, product_fee_description, product_from_location, - product_from_location_type, product_from_region_code, product_instance_family, - product_instance_type, product_instancesku, product_location, product_location_type, - product_operation, product_pricing_unit, product_product_family, product_region_code, - product_servicecode, product_sku, product_to_location, product_to_location_type, - product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, - reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, - reservation_effective_cost, reservation_end_time, reservation_modification_status, - reservation_net_amortized_upfront_cost_for_usage, - reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, - reservation_net_recurring_fee_for_usage, - reservation_net_unused_amortized_upfront_fee_for_billing_period, - reservation_net_unused_recurring_fee, reservation_net_upfront_value, - reservation_normalized_units_per_reservation, reservation_number_of_reservations, - reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, - reservation_start_time, reservation_subscription_id, - reservation_total_reserved_normalized_units, reservation_total_reserved_units, - reservation_units_per_reservation, - reservation_unused_amortized_upfront_fee_for_billing_period, - reservation_unused_normalized_unit_quantity, reservation_unused_quantity, - reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, - savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, - savings_plan_instance_type_family, - savings_plan_net_amortized_upfront_commitment_for_billing_period, - savings_plan_net_recurring_commitment_for_billing_period, - savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, - savings_plan_payment_option, savings_plan_purchase_term, - savings_plan_recurring_commitment_for_billing_period, savings_plan_region, - savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, - savings_plan_savings_plan_rate, savings_plan_start_time, - savings_plan_total_commitment_to_date, savings_plan_used_commitment, - split_line_item_actual_usage, split_line_item_net_split_cost, - split_line_item_net_unused_cost, split_line_item_parent_resource_id, - split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, - split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, - split_line_item_split_usage_ratio, split_line_item_unused_cost FROM COST_AND_USAGE_REPORT - TableConfigurations: - COST_AND_USAGE_REPORT: - INCLUDE_RESOURCES: "TRUE" - INCLUDE_SPLIT_COST_ALLOCATION_DATA: "TRUE" - TIME_GRANULARITY: "HOURLY" - DestinationConfigurations: - S3Destination: - S3Bucket: !Ref CloudCostBucketName - S3Region: !Ref CloudCostBucketRegion - S3Prefix: !Ref CloudCostReportPrefix - S3OutputConfigurations: - Compression: "PARQUET" - Format: "PARQUET" - Overwrite: "CREATE_NEW_REPORT" - OutputType: "CUSTOM" - Name: !Ref CloudCostReportName - RefreshCadence: - Frequency: "SYNCHRONOUS" - - CURWaitHandle: - Condition: ShouldCreateCloudCostReport - DependsOn: DDCURReportDefinition - Type: "AWS::CloudFormation::WaitConditionHandle" - - CUREmptyWaitHandle: - Type: "AWS::CloudFormation::WaitConditionHandle" - - CURWaitCondition: - Type: "AWS::CloudFormation::WaitCondition" - Properties: - Handle: - !If [ - ShouldCreateCloudCostReport, - !Ref CURWaitHandle, - !Ref CUREmptyWaitHandle, - ] - Timeout: "1" - Count: 0 - - # Lambda function to call Datadog API and configure CCM - DatadogCCMAPIHandlerLambdaExecutionRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: "/" - ManagedPolicyArns: - - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - - DatadogCCMAPICallFunction: - Type: AWS::Lambda::Function - Properties: - Description: "A function to call the Datadog API to configure CCM data export." - Role: !GetAtt DatadogCCMAPIHandlerLambdaExecutionRole.Arn - Handler: "index.handler" - LoggingConfig: - ApplicationLogLevel: "INFO" - LogFormat: "JSON" - Runtime: "python3.14" - Timeout: 30 - Code: - ZipFile: | - - - DatadogCCMConfiguration: - Type: Custom::DatadogCCMConfiguration - DependsOn: - - DatadogCCMIAMPolicy - - CURWaitCondition - Properties: - ServiceToken: !GetAtt DatadogCCMAPICallFunction.Arn - APIKey: !Ref DatadogApiKey - APPKey: !Ref DatadogAppKey - ApiURL: !Ref DatadogSite - AccountId: !Ref AWS::AccountId - BucketName: !Ref CloudCostBucketName - BucketRegion: !Ref CloudCostBucketRegion - ReportName: !Ref CloudCostReportName - ReportPrefix: !Ref CloudCostReportPrefix - -Outputs: - DatadogAccountUUID: - Description: The Datadog AWS Integration Account UUID - Value: !GetAtt DatadogCCMConfiguration.AccountUUID diff --git a/aws_cloud_cost_cur2/release.sh b/aws_cloud_cost_cur2/release.sh deleted file mode 100755 index b3acd5c9..00000000 --- a/aws_cloud_cost_cur2/release.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -# Usage: ./release.sh [--private] [--yes] - -set -e - -# Read the S3 bucket -if [ -z "$1" ]; then - echo "Must specify a S3 bucket to publish the template" - exit 1 -else - BUCKET=$1 -fi - -# Parse optional flags -PRIVATE_TEMPLATE=false -AUTO_YES=false -shift -while [[ $# -gt 0 ]]; do - case "$1" in - --private) - PRIVATE_TEMPLATE=true - shift - ;; - --yes) - AUTO_YES=true - shift - ;; - *) - echo "Unknown option: $1" - echo "Usage: ./release.sh [--private] [--yes]" - exit 1 - ;; - esac -done - -# Read the version -VERSION=$(head -n 1 version.txt) - -# Confirm the bucket for the current release doesn't already exist so we don't overwrite it -set +e -EXIT_CODE=0 -response=$(aws s3api head-object \ - --bucket "${BUCKET}" \ - --key "aws_cloud_cost_cur2/${VERSION}/main.yaml" > /dev/null 2>&1) - -if [[ ${?} -eq 0 ]]; then - echo "S3 bucket path ${BUCKET}/aws_cloud_cost_cur2/${VERSION} already exists. Please up the version." - exit 1 -fi -set -e - -# Confirm that the readme link points to the current release -set +e -response=$(grep -q "aws_cloud_cost_cur2/${VERSION}/main.yaml" README.md) -if [[ ${?} -ne 0 ]]; then - echo "README.md does not point to the current release. Please update the link." - exit 1 -fi -set -e - -# Confirm to proceed -for i in *.yaml; do - [ -f "$i" ] || break - echo "About to upload $i to s3://${BUCKET}/aws_cloud_cost_cur2/${VERSION}/$i" -done - -if [ "$AUTO_YES" = false ]; then - read -p "Continue (y/n)?" CONT - if [ "$CONT" != "y" ]; then - echo "Exiting" - exit 1 - fi -else - echo "Proceeding with upload (--yes flag provided)" -fi - -# Update bucket placeholder -cp main.yaml main.yaml.bak -perl -pi -e "s//${BUCKET}/g" main.yaml -perl -pi -e "s//${VERSION}/g" main.yaml - -# Replace ZIPFILE_PLACEHOLDER with the contents of the Python file -perl -i -pe ' - BEGIN { $p = do { local $/; } } - /^(\s+)/ && ( - $_ = join("\n", map { $1 . $_ } split(/\n/, $p)) . "\n" - ) -' main.yaml < datadog_ccm_api_call.py - -trap 'mv main.yaml.bak main.yaml' EXIT - -# Upload -if [ "$PRIVATE_TEMPLATE" = true ] ; then - aws s3 cp . s3://${BUCKET}/aws_cloud_cost_cur2/${VERSION} --recursive --exclude "*" --include "*.yaml" -else - aws s3 cp . s3://${BUCKET}/aws_cloud_cost_cur2/${VERSION} --recursive --exclude "*" --include "*.yaml" -fi -echo "Done uploading the template, and here is the CloudFormation quick launch URL" -echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloud-cost&templateURL=https://${BUCKET}.s3.amazonaws.com/aws_cloud_cost_cur2/${VERSION}/main.yaml" - -echo "Done!" diff --git a/aws_cloud_cost_cur2/version.txt b/aws_cloud_cost_cur2/version.txt deleted file mode 100644 index 77cada2e..00000000 --- a/aws_cloud_cost_cur2/version.txt +++ /dev/null @@ -1 +0,0 @@ -v0.0.6