From a253665c5ae9ad4d3a950f4025607fa61ce865f5 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:07:25 +0900 Subject: [PATCH 01/11] =?UTF-8?q?chore:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20secret=20=ED=8F=AC=EC=9D=B8=ED=84=B0=20?= =?UTF-8?q?=EA=B0=B1=EC=8B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: 부하 테스트 실행에 필요한 secret submodule 변경 커밋을 상위 인프라 저장소에 반영 --- config/secrets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/secrets b/config/secrets index f88a84c..b11ecad 160000 --- a/config/secrets +++ b/config/secrets @@ -1 +1 @@ -Subproject commit f88a84cdab72136d294614fd1e2c855c4a026c43 +Subproject commit b11ecadb9329eb4e84b93e98e94d8ab685605caa From d7456eccf62c1d73c0fd7cbf07dbfcaa806b3f7e Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:07:31 +0900 Subject: [PATCH 02/11] =?UTF-8?q?feat:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20RDS=20=EC=9D=B8=ED=94=84=EB=9D=BC=20?= =?UTF-8?q?=EA=B5=AC=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: 부하 테스트용 RDS, 보안 그룹, SSM datasource 파라미터를 Terraform으로 정의 - 상세내용: prod/stage EC2 보안 그룹에서 loadtest RDS 3306 접근을 허용하도록 구성 --- environment/load_test/main.tf | 145 ++++++++++++++++++++++++++++- environment/load_test/output.tf | 49 ++++++++++ environment/load_test/provider.tf | 19 ++++ environment/load_test/variables.tf | 77 ++++++++++++++- 4 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 environment/load_test/output.tf diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index 995074f..72b1c58 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -1 +1,144 @@ -# TODO:: 부하 테스트용 EC2 인스턴스 및 보안 그룹 리소스 정의 필요 +data "aws_vpc" "default" { + default = true +} + +data "aws_subnets" "default" { + filter { + name = "vpc-id" + values = [data.aws_vpc.default.id] + } +} + +data "aws_instance" "prod_api" { + filter { + name = "tag:Name" + values = [var.prod_api_instance_name] + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + +data "aws_instance" "stage_api" { + filter { + name = "tag:Name" + values = [var.stage_api_instance_name] + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + +data "aws_db_instance" "prod" { + db_instance_identifier = var.prod_rds_identifier +} + +data "aws_ssm_parameter" "db_root_username" { + name = var.load_test_db_username_parameter_name +} + +data "aws_ssm_parameter" "db_root_password" { + name = var.load_test_db_password_parameter_name + with_decryption = true +} + +locals { + db_root_username = data.aws_ssm_parameter.db_root_username.value + db_root_password = data.aws_ssm_parameter.db_root_password.value + + source_security_group_ids = setunion( + data.aws_instance.prod_api.vpc_security_group_ids, + data.aws_instance.stage_api.vpc_security_group_ids + ) +} + +resource "aws_security_group" "load_test_db" { + name = "sc-load-test-db-sg" + description = "Security group for load test RDS" + vpc_id = data.aws_vpc.default.id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "solid-connection-load-test-db-sg" + } +} + +resource "aws_security_group_rule" "load_test_db_mysql" { + for_each = local.source_security_group_ids + + type = "ingress" + description = "MySQL from prod/stage API server" + from_port = 3306 + to_port = 3306 + protocol = "tcp" + security_group_id = aws_security_group.load_test_db.id + source_security_group_id = each.value +} + +resource "aws_db_subnet_group" "load_test" { + name = "sc-load-test-db-subnet-group" + subnet_ids = data.aws_subnets.default.ids + + tags = { + Name = "solid-connection-load-test-db-subnet-group" + } +} + +resource "aws_db_instance" "load_test" { + identifier = var.rds_identifier + allocated_storage = var.allocated_storage + engine = "mysql" + engine_version = var.db_engine_version + instance_class = var.db_instance_class + db_name = var.db_name + username = local.db_root_username + password = local.db_root_password + parameter_group_name = var.db_parameter_group_name + db_subnet_group_name = aws_db_subnet_group.load_test.name + vpc_security_group_ids = [aws_security_group.load_test_db.id] + publicly_accessible = false + skip_final_snapshot = true + copy_tags_to_snapshot = true + deletion_protection = false + backup_retention_period = 0 + apply_immediately = true + storage_encrypted = true + kms_key_id = var.kms_key_arn + + tags = { + Name = var.rds_identifier + } +} + +resource "aws_ssm_parameter" "load_test_datasource_url" { + name = "${var.load_test_parameter_prefix}/spring.datasource.url" + type = "String" + value = "jdbc:mysql://${aws_db_instance.load_test.address}:${aws_db_instance.load_test.port}/${var.db_name}?serverTimezone=Asia/Seoul&characterEncoding=UTF-8" + overwrite = true +} + +resource "aws_ssm_parameter" "load_test_datasource_username" { + name = "${var.load_test_parameter_prefix}/spring.datasource.username" + type = "String" + value = local.db_root_username + overwrite = true +} + +resource "aws_ssm_parameter" "load_test_datasource_password" { + name = "${var.load_test_parameter_prefix}/spring.datasource.password" + type = "SecureString" + value = local.db_root_password + key_id = var.ssm_kms_key_id + overwrite = true + tier = "Standard" +} diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf new file mode 100644 index 0000000..55390ac --- /dev/null +++ b/environment/load_test/output.tf @@ -0,0 +1,49 @@ +output "load_test_rds_endpoint" { + description = "Load test RDS endpoint" + value = aws_db_instance.load_test.address +} + +output "load_test_rds_port" { + description = "Load test RDS port" + value = aws_db_instance.load_test.port +} + +output "load_test_rds_identifier" { + description = "Load test RDS identifier" + value = aws_db_instance.load_test.identifier +} + +output "load_test_db_name" { + description = "Load test database name" + value = var.db_name +} + +output "prod_rds_endpoint" { + description = "Prod RDS endpoint used as dump source" + value = data.aws_db_instance.prod.address +} + +output "prod_rds_port" { + description = "Prod RDS port" + value = data.aws_db_instance.prod.port +} + +output "prod_api_instance_id" { + description = "Prod API EC2 instance ID used to run migration commands" + value = data.aws_instance.prod_api.id +} + +output "stage_api_instance_id" { + description = "Stage API EC2 instance ID" + value = data.aws_instance.stage_api.id +} + +output "stage_api_public_ip" { + description = "Stage API EC2 public IP" + value = data.aws_instance.stage_api.public_ip +} + +output "load_test_ssm_parameter_prefix" { + description = "SSM Parameter Store prefix for load test datasource values" + value = var.load_test_parameter_prefix +} diff --git a/environment/load_test/provider.tf b/environment/load_test/provider.tf index 3c3f8d1..8b41b3d 100644 --- a/environment/load_test/provider.tf +++ b/environment/load_test/provider.tf @@ -1,3 +1,22 @@ +terraform { + required_version = ">= 1.10.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "solid-connection-tfstate" + key = "env/load_test/terraform.tfstate" + region = "ap-northeast-2" + use_lockfile = true + encrypt = true + } +} + provider "aws" { region = "ap-northeast-2" default_tags { diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index 6f74e1f..7d4d639 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -1 +1,76 @@ -# TODO:: 부하 테스트 인스턴스용 변수 정의 +variable "rds_identifier" { + description = "RDS identifier for load test" + type = string +} + +variable "db_instance_class" { + description = "RDS instance class for load test" + type = string +} + +variable "allocated_storage" { + description = "RDS storage in GiB" + type = number + default = 20 +} + +variable "db_engine_version" { + description = "MySQL engine version" + type = string +} + +variable "db_parameter_group_name" { + description = "MySQL parameter group name" + type = string +} + +variable "db_name" { + description = "Application database name" + type = string + default = "solid_connection" +} + +variable "load_test_db_username_parameter_name" { + description = "SSM parameter name containing the load test DB root username" + type = string +} + +variable "load_test_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the load test DB root password" + type = string +} + +variable "kms_key_arn" { + description = "KMS key ARN for RDS storage encryption" + type = string +} + +variable "ssm_kms_key_id" { + description = "KMS key ID or ARN for SSM SecureString. Null uses the AWS managed aws/ssm key." + type = string + default = null + nullable = true +} + +variable "prod_rds_identifier" { + description = "Source prod RDS identifier" + type = string +} + +variable "prod_api_instance_name" { + description = "Name tag of the prod API EC2 instance used to run dump/restore" + type = string + default = "solid-connection-server-prod" +} + +variable "stage_api_instance_name" { + description = "Name tag of the stage API EC2 instance that will connect to load test RDS" + type = string + default = "solid-connection-server-stage" +} + +variable "load_test_parameter_prefix" { + description = "SSM Parameter Store prefix for load test datasource values" + type = string + default = "/solid-connection/loadtest" +} From 95da845031ce00a915f26dfb6bea3d7a80d9c21d Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:07:36 +0900 Subject: [PATCH 03/11] =?UTF-8?q?feat:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=8B=A4=ED=96=89=20=EC=8A=A4=ED=81=AC?= =?UTF-8?q?=EB=A6=BD=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: start.sh에서 RDS 생성, stage 전환, prod 데이터 복사를 자동화 - 상세내용: stop.sh에서 stage 원복과 loadtest RDS destroy 흐름을 제공 - 상세내용: Windows와 macOS/Linux 실행 환경에서 사용할 bash 기반 절차를 문서화 --- scripts/load_test/README.md | 49 +++++++ scripts/load_test/start.sh | 264 ++++++++++++++++++++++++++++++++++++ scripts/load_test/stop.sh | 140 +++++++++++++++++++ 3 files changed, 453 insertions(+) create mode 100644 scripts/load_test/README.md create mode 100644 scripts/load_test/start.sh create mode 100644 scripts/load_test/stop.sh diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md new file mode 100644 index 0000000..2e4d129 --- /dev/null +++ b/scripts/load_test/README.md @@ -0,0 +1,49 @@ +# Load Test Automation + +This automation creates a temporary load test RDS instance, copies prod RDS data +into it, writes load test datasource values to Parameter Store, and optionally +stops/starts the stage application through SSM Run Command. + +## Flow + +1. `Start-LoadTest.ps1` runs `terraform apply` in `environment/load_test`. +2. Terraform creates the load test RDS and writes: + - `/solid-connection/loadtest/spring.datasource.url` + - `/solid-connection/loadtest/spring.datasource.username` + - `/solid-connection/loadtest/spring.datasource.password` +3. The script stores DB migration credentials in temporary SSM parameters. +4. The prod EC2 instance runs `mysqldump` against prod RDS and restores it into + the load test RDS. +5. The optional stage stop command can pause the stage app before the load test. +6. `Stop-LoadTest.ps1` can run an optional stage start command and then destroy + only the load test Terraform stack. + +## Example + +```bash +scripts/load_test/start.sh \ + --switch-stage-to-loadtest \ + --stage-ssh-key ./stage-key.pem +``` + +```bash +scripts/load_test/stop.sh \ + --restore-stage-dev \ + --stage-ssh-key ./stage-key.pem +``` + +## Notes + +- The prod and stage EC2 instances are looked up by their `Name` tags. +- Prod DB username/password are read from Parameter Store. The default paths are + `/solid-connection/prod/spring.datasource.username` and + `/solid-connection/prod/spring.datasource.password`. +- Load test DB username/password are also read from Parameter Store. The default + paths are `/solid-connection/loadtest/spring.datasource.username` and + `/solid-connection/loadtest/spring.datasource.password`. +- The load test RDS security group allows MySQL only from the security groups + attached to the prod and stage API EC2 instances. +- The prod EC2 instance must have SSM access and permission to read the temporary + migration parameters. +- Keep the real `load_test.tfvars` in the secret submodule or another ignored + local location. Do not commit it. diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh new file mode 100644 index 0000000..d962215 --- /dev/null +++ b/scripts/load_test/start.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash +set -euo pipefail + +TERRAFORM_DIR="environment/load_test" +VAR_FILE="../../config/secrets/load_test.tfvars" +DATABASE_NAME="solid_connection" +MIGRATION_PARAMETER_PREFIX="/solid-connection/loadtest/migration" +PROD_DB_USERNAME_PARAMETER="/solid-connection/prod/spring.datasource.username" +PROD_DB_PASSWORD_PARAMETER="/solid-connection/prod/spring.datasource.password" +LOADTEST_DB_USERNAME_PARAMETER="/solid-connection/loadtest/spring.datasource.username" +LOADTEST_DB_PASSWORD_PARAMETER="/solid-connection/loadtest/spring.datasource.password" +SWITCH_STAGE_TO_LOADTEST="false" +STAGE_SSH_USER="ubuntu" +STAGE_SSH_KEY="" +STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" +STAGE_COMPOSE_FILE="docker-compose.dev.yml" +SKIP_TERRAFORM_APPLY="false" +SKIP_DATA_COPY="false" + +usage() { + cat <<'EOF' +Usage: scripts/load_test/start.sh [options] + +Options: + --terraform-dir PATH Default: environment/load_test + --var-file PATH Default: ../../config/secrets/load_test.tfvars + --prod-db-username-parameter Default: /solid-connection/prod/spring.datasource.username + --prod-db-password-parameter Default: /solid-connection/prod/spring.datasource.password + --loadtest-db-username-parameter Default: /solid-connection/loadtest/spring.datasource.username + --loadtest-db-password-parameter Default: /solid-connection/loadtest/spring.datasource.password + --database-name VALUE Default: solid_connection + --migration-prefix VALUE Default: /solid-connection/loadtest/migration + --switch-stage-to-loadtest Restart stage app over SSH with dev,loadtest profiles + --stage-ssh-user VALUE Default: ubuntu + --stage-ssh-key PATH Required with --switch-stage-to-loadtest + --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev + --stage-compose-file VALUE Default: docker-compose.dev.yml + --skip-terraform-apply + --skip-data-copy + -h, --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --var-file) VAR_FILE="$2"; shift 2 ;; + --prod-db-username-parameter) PROD_DB_USERNAME_PARAMETER="$2"; shift 2 ;; + --prod-db-password-parameter) PROD_DB_PASSWORD_PARAMETER="$2"; shift 2 ;; + --loadtest-db-username-parameter) LOADTEST_DB_USERNAME_PARAMETER="$2"; shift 2 ;; + --loadtest-db-password-parameter) LOADTEST_DB_PASSWORD_PARAMETER="$2"; shift 2 ;; + --database-name) DATABASE_NAME="$2"; shift 2 ;; + --migration-prefix) MIGRATION_PARAMETER_PREFIX="$2"; shift 2 ;; + --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; + --stage-ssh-user) STAGE_SSH_USER="$2"; shift 2 ;; + --stage-ssh-key) STAGE_SSH_KEY="$2"; shift 2 ;; + --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; + --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; + --skip-data-copy) SKIP_DATA_COPY="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +require_value() { + local name="$1" + local value="$2" + if [[ -z "$value" ]]; then + echo "Missing required option: $name" >&2 + exit 1 + fi +} + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Required command not found: $1" >&2 + exit 1 + fi +} + +require_command terraform +require_command aws +require_command jq +require_command ssh + +tf_output() { + terraform -chdir="$TERRAFORM_DIR" output -raw "$1" +} + +send_ssm_command() { + local instance_id="$1" + local comment="$2" + local commands_json="$3" + + local command_id + command_id="$(aws ssm send-command \ + --instance-ids "$instance_id" \ + --document-name "AWS-RunShellScript" \ + --comment "$comment" \ + --parameters "$commands_json" \ + --query "Command.CommandId" \ + --output text)" + + local status + while true; do + sleep 5 + status="$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "Status" \ + --output text 2>/dev/null || true)" + + case "$status" in + Pending|InProgress|Delayed|"") continue ;; + Success) break ;; + *) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command failed with status $status: $comment" >&2 + exit 1 + ;; + esac + done +} + +delete_temp_parameters() { + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" >/dev/null 2>&1 || true + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" >/dev/null 2>&1 || true + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-username" >/dev/null 2>&1 || true + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-password" >/dev/null 2>&1 || true +} + +if [[ "$SKIP_TERRAFORM_APPLY" != "true" ]]; then + terraform -chdir="$TERRAFORM_DIR" init + terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" +fi + +prod_instance_id="$(tf_output prod_api_instance_id)" +stage_instance_id="$(tf_output stage_api_instance_id)" +stage_public_ip="$(tf_output stage_api_public_ip)" +prod_endpoint="$(tf_output prod_rds_endpoint)" +prod_port="$(tf_output prod_rds_port)" +loadtest_endpoint="$(tf_output load_test_rds_endpoint)" +loadtest_port="$(tf_output load_test_rds_port)" + +if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then + require_value "--stage-ssh-key" "$STAGE_SSH_KEY" + + ssh -i "$STAGE_SSH_KEY" \ + -o StrictHostKeyChecking=no \ + "$STAGE_SSH_USER@$stage_public_ip" \ + "APP_DIR='$STAGE_APP_DIR' COMPOSE_FILE='$STAGE_COMPOSE_FILE' bash -s" <<'REMOTE' +set -euo pipefail + +cd "$APP_DIR" + +CURRENT_IMAGE="$(docker inspect -f '{{.Config.Image}}' solid-connection-dev 2>/dev/null || true)" +if [[ -z "$CURRENT_IMAGE" ]]; then + echo "solid-connection-dev container is not running; cannot infer image tag" >&2 + exit 1 +fi + +OWNER_LOWERCASE="$(echo "$CURRENT_IMAGE" | sed -E 's#^ghcr.io/([^/]+)/.*#\1#')" +IMAGE_TAG="$(echo "$CURRENT_IMAGE" | sed -E 's#.*:([^:]+)$#\1#')" + +cat > docker-compose.loadtest.override.yml <<'YAML' +services: + solid-connection-dev: + environment: + - SPRING_PROFILES_ACTIVE=dev,loadtest + - AWS_REGION=ap-northeast-2 + - SPRING_DATA_REDIS_HOST=127.0.0.1 + - SPRING_DATA_REDIS_PORT=6379 +YAML + +docker compose -f "$COMPOSE_FILE" -f docker-compose.loadtest.override.yml down || true +OWNER_LOWERCASE="$OWNER_LOWERCASE" IMAGE_TAG="$IMAGE_TAG" \ + docker compose -f "$COMPOSE_FILE" -f docker-compose.loadtest.override.yml up -d solid-connection-dev +REMOTE +fi + +if [[ "$SKIP_DATA_COPY" != "true" ]]; then + trap delete_temp_parameters EXIT + + prod_db_username="$(aws ssm get-parameter \ + --name "$PROD_DB_USERNAME_PARAMETER" \ + --query "Parameter.Value" \ + --output text)" + + prod_db_password="$(aws ssm get-parameter \ + --name "$PROD_DB_PASSWORD_PARAMETER" \ + --with-decryption \ + --query "Parameter.Value" \ + --output text)" + + loadtest_db_username="$(aws ssm get-parameter \ + --name "$LOADTEST_DB_USERNAME_PARAMETER" \ + --query "Parameter.Value" \ + --output text)" + + loadtest_db_password="$(aws ssm get-parameter \ + --name "$LOADTEST_DB_PASSWORD_PARAMETER" \ + --with-decryption \ + --query "Parameter.Value" \ + --output text)" + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" \ + --type String \ + --value "$prod_db_username" \ + --overwrite >/dev/null + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" \ + --type SecureString \ + --value "$prod_db_password" \ + --overwrite >/dev/null + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-username" \ + --type String \ + --value "$loadtest_db_username" \ + --overwrite >/dev/null + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-password" \ + --type SecureString \ + --value "$loadtest_db_password" \ + --overwrite >/dev/null + + copy_commands_json="$(jq -cn \ + --arg prefix "$MIGRATION_PARAMETER_PREFIX" \ + --arg prod_endpoint "$prod_endpoint" \ + --arg prod_port "$prod_port" \ + --arg loadtest_endpoint "$loadtest_endpoint" \ + --arg loadtest_port "$loadtest_port" \ + --arg database "$DATABASE_NAME" \ + '{ + commands: [ + "set -euo pipefail", + "export DEBIAN_FRONTEND=noninteractive", + "if ! command -v mysqldump >/dev/null 2>&1 || ! command -v mysql >/dev/null 2>&1; then sudo apt-get update && sudo apt-get install -y mysql-client; fi", + "PROD_USER=$(aws ssm get-parameter --name \($prefix)/prod-db-username --query Parameter.Value --output text)", + "PROD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/prod-db-password --with-decryption --query Parameter.Value --output text)", + "LOAD_USER=$(aws ssm get-parameter --name \($prefix)/loadtest-db-username --query Parameter.Value --output text)", + "LOAD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/loadtest-db-password --with-decryption --query Parameter.Value --output text)", + "DUMP_FILE=/tmp/solid-connection-loadtest-$(date +%Y%m%d%H%M%S).sql.gz", + "MYSQL_PWD=\"$PROD_PASSWORD\" mysqldump --single-transaction --set-gtid-purged=OFF --column-statistics=0 -h \($prod_endpoint) -P \($prod_port) -u \"$PROD_USER\" \($database) | gzip > \"$DUMP_FILE\"", + "MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" -e \"DROP DATABASE IF EXISTS \\\`\($database)\\\`; CREATE DATABASE \\\`\($database)\\\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;\"", + "gunzip -c \"$DUMP_FILE\" | MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" \($database)", + "rm -f \"$DUMP_FILE\"" + ] + }')" + + send_ssm_command "$prod_instance_id" "Copy prod RDS data to load test RDS" "$copy_commands_json" +fi + +echo "Load test environment is ready." +echo "RDS endpoint: ${loadtest_endpoint}:${loadtest_port}" +echo "Stage instance: ${stage_instance_id}" +echo "Stage public IP: ${stage_public_ip}" diff --git a/scripts/load_test/stop.sh b/scripts/load_test/stop.sh new file mode 100644 index 0000000..f0857ec --- /dev/null +++ b/scripts/load_test/stop.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +set -euo pipefail + +TERRAFORM_DIR="environment/load_test" +VAR_FILE="../../config/secrets/load_test.tfvars" +STAGE_START_COMMAND="" +RESTORE_STAGE_DEV="false" +STAGE_SSH_USER="ubuntu" +STAGE_SSH_KEY="" +STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" +STAGE_COMPOSE_FILE="docker-compose.dev.yml" +SKIP_TERRAFORM_DESTROY="false" + +usage() { + cat <<'EOF' +Usage: scripts/load_test/stop.sh [options] + +Options: + --terraform-dir PATH Default: environment/load_test + --var-file PATH Default: ../../config/secrets/load_test.tfvars + --restore-stage-dev Restart stage app over SSH with dev profile + --stage-ssh-user VALUE Default: ubuntu + --stage-ssh-key PATH Required with --restore-stage-dev + --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev + --stage-compose-file VALUE Default: docker-compose.dev.yml + --skip-terraform-destroy + -h, --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --var-file) VAR_FILE="$2"; shift 2 ;; + --restore-stage-dev) RESTORE_STAGE_DEV="true"; shift ;; + --stage-ssh-user) STAGE_SSH_USER="$2"; shift 2 ;; + --stage-ssh-key) STAGE_SSH_KEY="$2"; shift 2 ;; + --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; + --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --skip-terraform-destroy) SKIP_TERRAFORM_DESTROY="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Required command not found: $1" >&2 + exit 1 + fi +} + +require_command terraform +require_command aws +require_command jq +require_command ssh + +require_value() { + local name="$1" + local value="$2" + if [[ -z "$value" ]]; then + echo "Missing required option: $name" >&2 + exit 1 + fi +} + +tf_output() { + terraform -chdir="$TERRAFORM_DIR" output -raw "$1" +} + +send_ssm_command() { + local instance_id="$1" + local comment="$2" + local commands_json="$3" + + local command_id + command_id="$(aws ssm send-command \ + --instance-ids "$instance_id" \ + --document-name "AWS-RunShellScript" \ + --comment "$comment" \ + --parameters "$commands_json" \ + --query "Command.CommandId" \ + --output text)" + + local status + while true; do + sleep 5 + status="$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "Status" \ + --output text 2>/dev/null || true)" + + case "$status" in + Pending|InProgress|Delayed|"") continue ;; + Success) break ;; + *) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command failed with status $status: $comment" >&2 + exit 1 + ;; + esac + done +} + +if [[ "$RESTORE_STAGE_DEV" == "true" ]]; then + require_value "--stage-ssh-key" "$STAGE_SSH_KEY" + stage_public_ip="$(tf_output stage_api_public_ip)" + + ssh -i "$STAGE_SSH_KEY" \ + -o StrictHostKeyChecking=no \ + "$STAGE_SSH_USER@$stage_public_ip" \ + "APP_DIR='$STAGE_APP_DIR' COMPOSE_FILE='$STAGE_COMPOSE_FILE' bash -s" <<'REMOTE' +set -euo pipefail + +cd "$APP_DIR" + +CURRENT_IMAGE="$(docker inspect -f '{{.Config.Image}}' solid-connection-dev 2>/dev/null || true)" +if [[ -z "$CURRENT_IMAGE" ]]; then + echo "solid-connection-dev container is not running; cannot infer image tag" >&2 + exit 1 +fi + +OWNER_LOWERCASE="$(echo "$CURRENT_IMAGE" | sed -E 's#^ghcr.io/([^/]+)/.*#\1#')" +IMAGE_TAG="$(echo "$CURRENT_IMAGE" | sed -E 's#.*:([^:]+)$#\1#')" + +rm -f docker-compose.loadtest.override.yml +docker compose -f "$COMPOSE_FILE" down || true +OWNER_LOWERCASE="$OWNER_LOWERCASE" IMAGE_TAG="$IMAGE_TAG" docker compose -f "$COMPOSE_FILE" up -d +REMOTE +fi + +if [[ "$SKIP_TERRAFORM_DESTROY" != "true" ]]; then + terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" +fi + +echo "Load test environment has been stopped." From bdfb439882efaf23319beb346213199019404a3a Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:47:00 +0900 Subject: [PATCH 04/11] =?UTF-8?q?feat:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20GitHub=20Actions=20=EC=8B=A4=ED=96=89=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: workflow_dispatch로 부하 테스트 시작과 종료를 수동 실행할 수 있도록 워크플로우 추가 - 상세내용: stage 서버 전환과 원복을 SSH 대신 SSM RunCommand로 수행하도록 변경 - 상세내용: SSH key 입력 없이 OIDC 기반 AWS Role과 GH_PAT submodule checkout 흐름을 사용하도록 문서화 --- .github/workflows/load-test-start.yml | 65 +++++++++++++++++++++++ .github/workflows/load-test-stop.yml | 65 +++++++++++++++++++++++ scripts/load_test/README.md | 74 ++++++++++++++++----------- scripts/load_test/start.sh | 58 +++++++-------------- scripts/load_test/stop.sh | 65 +++++++++-------------- 5 files changed, 214 insertions(+), 113 deletions(-) create mode 100644 .github/workflows/load-test-start.yml create mode 100644 .github/workflows/load-test-stop.yml diff --git a/.github/workflows/load-test-start.yml b/.github/workflows/load-test-start.yml new file mode 100644 index 0000000..80125e4 --- /dev/null +++ b/.github/workflows/load-test-start.yml @@ -0,0 +1,65 @@ +name: Load Test Start + +on: + workflow_dispatch: + inputs: + switch_stage_to_loadtest: + description: "Restart stage app with dev,loadtest profiles" + required: true + default: true + type: boolean + copy_prod_data: + description: "Copy prod RDS data to load test RDS" + required: true + default: true + type: boolean + +permissions: + id-token: write + contents: read + +concurrency: + group: load-test-environment + cancel-in-progress: false + +env: + TF_VERSION: "1.10.5" + +jobs: + start: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ap-northeast-2 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Start load test environment + run: | + args=() + + if [ "${{ inputs.switch_stage_to_loadtest }}" = "true" ]; then + args+=(--switch-stage-to-loadtest) + fi + + if [ "${{ inputs.copy_prod_data }}" != "true" ]; then + args+=(--skip-data-copy) + fi + + bash scripts/load_test/start.sh "${args[@]}" diff --git a/.github/workflows/load-test-stop.yml b/.github/workflows/load-test-stop.yml new file mode 100644 index 0000000..9e5fba7 --- /dev/null +++ b/.github/workflows/load-test-stop.yml @@ -0,0 +1,65 @@ +name: Load Test Stop + +on: + workflow_dispatch: + inputs: + restore_stage_dev: + description: "Restart stage app with dev profile" + required: true + default: true + type: boolean + destroy_rds: + description: "Destroy load test Terraform stack" + required: true + default: true + type: boolean + +permissions: + id-token: write + contents: read + +concurrency: + group: load-test-environment + cancel-in-progress: false + +env: + TF_VERSION: "1.10.5" + +jobs: + stop: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ap-northeast-2 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Stop load test environment + run: | + args=() + + if [ "${{ inputs.restore_stage_dev }}" = "true" ]; then + args+=(--restore-stage-dev) + fi + + if [ "${{ inputs.destroy_rds }}" != "true" ]; then + args+=(--skip-terraform-destroy) + fi + + bash scripts/load_test/stop.sh "${args[@]}" diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 2e4d129..0283669 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -1,49 +1,61 @@ # Load Test Automation This automation creates a temporary load test RDS instance, copies prod RDS data -into it, writes load test datasource values to Parameter Store, and optionally -stops/starts the stage application through SSM Run Command. +into it, writes load test datasource values to Parameter Store, and switches the +stage application through SSM Run Command. -## Flow +## GitHub Actions Flow -1. `Start-LoadTest.ps1` runs `terraform apply` in `environment/load_test`. -2. Terraform creates the load test RDS and writes: +### Start + +1. Open **Actions > Load Test Start**. +2. Click **Run workflow**. +3. Keep `switch_stage_to_loadtest` enabled to restart stage with + `dev,loadtest` profiles. +4. Keep `copy_prod_data` enabled to copy prod RDS data into the load test RDS. + +The workflow runs `scripts/load_test/start.sh`. + +### Stop + +1. Open **Actions > Load Test Stop**. +2. Click **Run workflow**. +3. Keep `restore_stage_dev` enabled to restart stage with the normal dev + compose configuration. +4. Keep `destroy_rds` enabled to destroy the load test Terraform stack. + +The workflow runs `scripts/load_test/stop.sh`. + +## What Start Does + +1. Runs `terraform apply` in `environment/load_test`. +2. Creates the load test RDS and writes: - `/solid-connection/loadtest/spring.datasource.url` - `/solid-connection/loadtest/spring.datasource.username` - `/solid-connection/loadtest/spring.datasource.password` -3. The script stores DB migration credentials in temporary SSM parameters. -4. The prod EC2 instance runs `mysqldump` against prod RDS and restores it into - the load test RDS. -5. The optional stage stop command can pause the stage app before the load test. -6. `Stop-LoadTest.ps1` can run an optional stage start command and then destroy - only the load test Terraform stack. - -## Example - -```bash -scripts/load_test/start.sh \ - --switch-stage-to-loadtest \ - --stage-ssh-key ./stage-key.pem -``` - -```bash -scripts/load_test/stop.sh \ - --restore-stage-dev \ - --stage-ssh-key ./stage-key.pem -``` +3. Switches the stage app to `dev,loadtest` profiles through SSM Run Command. +4. Stores DB migration credentials in temporary SSM parameters. +5. Runs `mysqldump` on the prod EC2 instance through SSM Run Command and + restores the dump into the load test RDS. +6. Deletes the temporary migration parameters. + +## What Stop Does + +1. Restores the stage app to the normal dev compose configuration through SSM + Run Command. +2. Runs `terraform destroy` for the load test Terraform stack. ## Notes +- GitHub Actions uses `AWS_ROLE_ARN` through OIDC. +- Private submodule checkout uses `GH_PAT`. +- No SSH private key is required for load test start/stop. - The prod and stage EC2 instances are looked up by their `Name` tags. - Prod DB username/password are read from Parameter Store. The default paths are `/solid-connection/prod/spring.datasource.username` and `/solid-connection/prod/spring.datasource.password`. -- Load test DB username/password are also read from Parameter Store. The default - paths are `/solid-connection/loadtest/spring.datasource.username` and +- Load test DB username/password are read from Parameter Store. The default paths + are `/solid-connection/loadtest/spring.datasource.username` and `/solid-connection/loadtest/spring.datasource.password`. - The load test RDS security group allows MySQL only from the security groups attached to the prod and stage API EC2 instances. -- The prod EC2 instance must have SSM access and permission to read the temporary - migration parameters. -- Keep the real `load_test.tfvars` in the secret submodule or another ignored - local location. Do not commit it. diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh index d962215..1f9c75a 100644 --- a/scripts/load_test/start.sh +++ b/scripts/load_test/start.sh @@ -10,8 +10,6 @@ PROD_DB_PASSWORD_PARAMETER="/solid-connection/prod/spring.datasource.password" LOADTEST_DB_USERNAME_PARAMETER="/solid-connection/loadtest/spring.datasource.username" LOADTEST_DB_PASSWORD_PARAMETER="/solid-connection/loadtest/spring.datasource.password" SWITCH_STAGE_TO_LOADTEST="false" -STAGE_SSH_USER="ubuntu" -STAGE_SSH_KEY="" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" SKIP_TERRAFORM_APPLY="false" @@ -30,9 +28,7 @@ Options: --loadtest-db-password-parameter Default: /solid-connection/loadtest/spring.datasource.password --database-name VALUE Default: solid_connection --migration-prefix VALUE Default: /solid-connection/loadtest/migration - --switch-stage-to-loadtest Restart stage app over SSH with dev,loadtest profiles - --stage-ssh-user VALUE Default: ubuntu - --stage-ssh-key PATH Required with --switch-stage-to-loadtest + --switch-stage-to-loadtest Restart stage app through SSM with dev,loadtest profiles --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml --skip-terraform-apply @@ -52,8 +48,6 @@ while [[ $# -gt 0 ]]; do --database-name) DATABASE_NAME="$2"; shift 2 ;; --migration-prefix) MIGRATION_PARAMETER_PREFIX="$2"; shift 2 ;; --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; - --stage-ssh-user) STAGE_SSH_USER="$2"; shift 2 ;; - --stage-ssh-key) STAGE_SSH_KEY="$2"; shift 2 ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; @@ -82,7 +76,6 @@ require_command() { require_command terraform require_command aws require_command jq -require_command ssh tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" @@ -147,39 +140,24 @@ loadtest_endpoint="$(tf_output load_test_rds_endpoint)" loadtest_port="$(tf_output load_test_rds_port)" if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then - require_value "--stage-ssh-key" "$STAGE_SSH_KEY" - - ssh -i "$STAGE_SSH_KEY" \ - -o StrictHostKeyChecking=no \ - "$STAGE_SSH_USER@$stage_public_ip" \ - "APP_DIR='$STAGE_APP_DIR' COMPOSE_FILE='$STAGE_COMPOSE_FILE' bash -s" <<'REMOTE' -set -euo pipefail - -cd "$APP_DIR" - -CURRENT_IMAGE="$(docker inspect -f '{{.Config.Image}}' solid-connection-dev 2>/dev/null || true)" -if [[ -z "$CURRENT_IMAGE" ]]; then - echo "solid-connection-dev container is not running; cannot infer image tag" >&2 - exit 1 -fi - -OWNER_LOWERCASE="$(echo "$CURRENT_IMAGE" | sed -E 's#^ghcr.io/([^/]+)/.*#\1#')" -IMAGE_TAG="$(echo "$CURRENT_IMAGE" | sed -E 's#.*:([^:]+)$#\1#')" - -cat > docker-compose.loadtest.override.yml <<'YAML' -services: - solid-connection-dev: - environment: - - SPRING_PROFILES_ACTIVE=dev,loadtest - - AWS_REGION=ap-northeast-2 - - SPRING_DATA_REDIS_HOST=127.0.0.1 - - SPRING_DATA_REDIS_PORT=6379 -YAML + stage_commands_json="$(jq -cn \ + --arg app_dir "$STAGE_APP_DIR" \ + --arg compose_file "$STAGE_COMPOSE_FILE" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($app_dir)", + "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", + "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", + "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", + "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", + "cat > docker-compose.loadtest.override.yml <<'\''YAML'\''\nservices:\n solid-connection-dev:\n environment:\n - SPRING_PROFILES_ACTIVE=dev,loadtest\n - AWS_REGION=ap-northeast-2\n - SPRING_DATA_REDIS_HOST=127.0.0.1\n - SPRING_DATA_REDIS_PORT=6379\nYAML", + "docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml down || true", + "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml up -d solid-connection-dev" + ] + }')" -docker compose -f "$COMPOSE_FILE" -f docker-compose.loadtest.override.yml down || true -OWNER_LOWERCASE="$OWNER_LOWERCASE" IMAGE_TAG="$IMAGE_TAG" \ - docker compose -f "$COMPOSE_FILE" -f docker-compose.loadtest.override.yml up -d solid-connection-dev -REMOTE + send_ssm_command "$stage_instance_id" "Switch stage app to load test datasource" "$stage_commands_json" fi if [[ "$SKIP_DATA_COPY" != "true" ]]; then diff --git a/scripts/load_test/stop.sh b/scripts/load_test/stop.sh index f0857ec..62687fb 100644 --- a/scripts/load_test/stop.sh +++ b/scripts/load_test/stop.sh @@ -3,10 +3,7 @@ set -euo pipefail TERRAFORM_DIR="environment/load_test" VAR_FILE="../../config/secrets/load_test.tfvars" -STAGE_START_COMMAND="" RESTORE_STAGE_DEV="false" -STAGE_SSH_USER="ubuntu" -STAGE_SSH_KEY="" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" SKIP_TERRAFORM_DESTROY="false" @@ -18,9 +15,7 @@ Usage: scripts/load_test/stop.sh [options] Options: --terraform-dir PATH Default: environment/load_test --var-file PATH Default: ../../config/secrets/load_test.tfvars - --restore-stage-dev Restart stage app over SSH with dev profile - --stage-ssh-user VALUE Default: ubuntu - --stage-ssh-key PATH Required with --restore-stage-dev + --restore-stage-dev Restart stage app through SSM with dev profile --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml --skip-terraform-destroy @@ -33,8 +28,6 @@ while [[ $# -gt 0 ]]; do --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; --var-file) VAR_FILE="$2"; shift 2 ;; --restore-stage-dev) RESTORE_STAGE_DEV="true"; shift ;; - --stage-ssh-user) STAGE_SSH_USER="$2"; shift 2 ;; - --stage-ssh-key) STAGE_SSH_KEY="$2"; shift 2 ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; --skip-terraform-destroy) SKIP_TERRAFORM_DESTROY="true"; shift ;; @@ -53,16 +46,6 @@ require_command() { require_command terraform require_command aws require_command jq -require_command ssh - -require_value() { - local name="$1" - local value="$2" - if [[ -z "$value" ]]; then - echo "Missing required option: $name" >&2 - exit 1 - fi -} tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" @@ -106,31 +89,29 @@ send_ssm_command() { done } -if [[ "$RESTORE_STAGE_DEV" == "true" ]]; then - require_value "--stage-ssh-key" "$STAGE_SSH_KEY" - stage_public_ip="$(tf_output stage_api_public_ip)" - - ssh -i "$STAGE_SSH_KEY" \ - -o StrictHostKeyChecking=no \ - "$STAGE_SSH_USER@$stage_public_ip" \ - "APP_DIR='$STAGE_APP_DIR' COMPOSE_FILE='$STAGE_COMPOSE_FILE' bash -s" <<'REMOTE' -set -euo pipefail - -cd "$APP_DIR" - -CURRENT_IMAGE="$(docker inspect -f '{{.Config.Image}}' solid-connection-dev 2>/dev/null || true)" -if [[ -z "$CURRENT_IMAGE" ]]; then - echo "solid-connection-dev container is not running; cannot infer image tag" >&2 - exit 1 -fi - -OWNER_LOWERCASE="$(echo "$CURRENT_IMAGE" | sed -E 's#^ghcr.io/([^/]+)/.*#\1#')" -IMAGE_TAG="$(echo "$CURRENT_IMAGE" | sed -E 's#.*:([^:]+)$#\1#')" +terraform -chdir="$TERRAFORM_DIR" init -rm -f docker-compose.loadtest.override.yml -docker compose -f "$COMPOSE_FILE" down || true -OWNER_LOWERCASE="$OWNER_LOWERCASE" IMAGE_TAG="$IMAGE_TAG" docker compose -f "$COMPOSE_FILE" up -d -REMOTE +if [[ "$RESTORE_STAGE_DEV" == "true" ]]; then + stage_instance_id="$(tf_output stage_api_instance_id)" + + stage_commands_json="$(jq -cn \ + --arg app_dir "$STAGE_APP_DIR" \ + --arg compose_file "$STAGE_COMPOSE_FILE" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($app_dir)", + "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", + "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", + "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", + "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", + "rm -f docker-compose.loadtest.override.yml", + "docker compose -f \($compose_file) down || true", + "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) up -d" + ] + }')" + + send_ssm_command "$stage_instance_id" "Restore stage app to dev datasource" "$stage_commands_json" fi if [[ "$SKIP_TERRAFORM_DESTROY" != "true" ]]; then From 9ef254a409cb20073077f84cbf7e23a1618b12d7 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 01:54:49 +0900 Subject: [PATCH 05/11] =?UTF-8?q?feat:=20stage=20=EC=83=9D=EC=84=B1=20?= =?UTF-8?q?=EC=8B=9C=20k6=20=ED=8C=8C=EC=9D=BC=20=EB=B0=B0=EC=B9=98=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: monitor repo의 k6 파일을 infra repo에 포함해 stage EC2 cloud-init에서 배치하도록 구성 - 상세내용: app_stack module에 k6 파일 배치 옵션을 추가하고 stage 환경에서만 활성화 - 상세내용: 부하 테스트 README를 한글로 변경하고 GitHub Actions 실행 흐름을 정리 --- config/load-test/k6/createPost.json | 7 + config/load-test/k6/script/set-load-test.sh | 64 ++++ config/load-test/k6/set_up_xk6.sh | 53 +++ config/load-test/k6/updatePost.json | 5 + config/load-test/k6/whole-user-flow.js | 372 ++++++++++++++++++++ environment/stage/main.tf | 12 +- modules/app_stack/ec2.tf | 63 +++- modules/app_stack/variables.tf | 14 +- scripts/load_test/README.md | 109 +++--- 9 files changed, 639 insertions(+), 60 deletions(-) create mode 100644 config/load-test/k6/createPost.json create mode 100644 config/load-test/k6/script/set-load-test.sh create mode 100644 config/load-test/k6/set_up_xk6.sh create mode 100644 config/load-test/k6/updatePost.json create mode 100644 config/load-test/k6/whole-user-flow.js diff --git a/config/load-test/k6/createPost.json b/config/load-test/k6/createPost.json new file mode 100644 index 0000000..e08b0d2 --- /dev/null +++ b/config/load-test/k6/createPost.json @@ -0,0 +1,7 @@ +{ + "boardCode": "FREE", + "postCategory": "자유", + "title": "수강신청 어떻게 하나요?", + "content": "수강신청 방법을 잘 모르겠어요.", + "isQuestion": false +} diff --git a/config/load-test/k6/script/set-load-test.sh b/config/load-test/k6/script/set-load-test.sh new file mode 100644 index 0000000..b401b2d --- /dev/null +++ b/config/load-test/k6/script/set-load-test.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +# 작업 디렉터리 설정 +WORKDIR="/home/ubuntu" + +####################################################################### +# set-load-test.sh +# 사용 예: SQL_FILE_BASENAME=regular.sql ./set-load-test.sh +####################################################################### + +# 0. 필수 값 검증 +SQL_BASENAME="${SQL_FILE_BASENAME:-regular.sql}" +if [[ -z "$SQL_BASENAME" ]]; then + echo "❌ 사용할 덤프 파일이 입력되지 않았습니다." >&2 + echo "SQL_FILE_BASENAME={dump 파일 이름} ./set-load-test.sh 형식으로 인자를 전달해야합니다." >&2 + exit 1 +fi + +SQL_SRC="$WORKDIR/load-test-setting/db/${SQL_BASENAME}" +if [[ ! -f "$SQL_SRC" ]]; then + echo "❌ 덤프 파일이 파일 시스템에 존재하지 않습니다: $SQL_SRC" >&2 + exit 2 +fi + +# 1. 기존 어플리케이션, DB 중지 +docker compose -f "$WORKDIR/solid-connection-dev/docker-compose.dev.yml" down \ + || { echo "❌ 어플리케이션 도커 중지 실패: $WORKDIR/solid-connection-dev/docker-compose.dev.yml" >&2; exit 3; } +docker compose -f "$WORKDIR/mysql/docker-compose.mysql.yml" down \ + || { echo "❌ MySQL 도커 중지 실패: $WORKDIR/mysql/docker-compose.mysql.yml" >&2; exit 4; } + +# 2. 부하 테스트용 DB 실행 +docker compose -f "$WORKDIR/load-test-setting/docker-compose.load-test.yml" up -d \ + || { echo "❌ 부하 테스트용 DB 실행 실패: docker-compose.load-test.yml" >&2; exit 5; } + +# 3. MySQL 준비 대기 (최대 30초) +CONTAINER_NAME="load-test-db" +echo "⏳ MySQL이 준비될 때까지 대기 중 (최대 30초)..." +start_time=$(date +%s) +while ! docker exec "$CONTAINER_NAME" sh -c 'mysqladmin ping -h "127.0.0.1" --silent'; do + elapsed=$(( $(date +%s) - start_time )) + if [[ $elapsed -ge 30 ]]; then + echo "❌ MySQL 준비 시간 초과 (30초)" >&2 + exit 1 + fi + printf "." + sleep 1 +done +echo "✔️ MySQL 준비 완료." + +# 4. dump 주입 +echo "📥 dump 파일 복사 → 컨테이너: $CONTAINER_NAME" +docker cp "$SQL_SRC" "${CONTAINER_NAME}:/tmp/dump.sql" \ + || { echo "❌ dump 복사 실패: $SQL_SRC → $CONTAINER_NAME:/tmp/dump.sql" >&2; exit 6; } + +echo "⚙️ dump 이식" +docker exec -i "$CONTAINER_NAME" sh -c 'mysql -u root -proot < /tmp/dump.sql' \ + || { echo "❌ dump 이식 실패: 컨테이너 $CONTAINER_NAME" >&2; exit 7; } + +# 5. 어플리케이션 다시 실행 +docker compose -f "$WORKDIR/solid-connection-dev/docker-compose.dev.yml" up -d \ + || { echo "❌ 어플리케이션 재시작 실패: $WORKDIR/solid-connection-dev/docker-compose.dev.yml" >&2; exit 8; } + +echo "✅ 부하 테스트용 DB에 연결된 어플리케이션 실행 완료!" diff --git a/config/load-test/k6/set_up_xk6.sh b/config/load-test/k6/set_up_xk6.sh new file mode 100644 index 0000000..005fb3d --- /dev/null +++ b/config/load-test/k6/set_up_xk6.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -euo pipefail + +# 에러 발생 시 메시지 출력 +trap 'echo "❌ 오류 발생! 스크립트 실행이 중단되었습니다." >&2' ERR + +export GO_VERSION=1.22.2 +export BASE_DIR=/home/ubuntu/solid-connection-load-test/k6 +export GOROOT=${BASE_DIR}/go +export GOPATH=${BASE_DIR}/go-workspace +export PATH=$PATH:$GOROOT/bin:$GOPATH/bin +export XK6_BIN=${GOPATH}/bin/xk6 +export K6_OUT=xk6-prometheus-rw +export K6_PROMETHEUS_RW_SERVER_URL=http://132.145.83.182:9090/api/v1/write +export K6_PROMETHEUS_RW_TREND_STATS="p(90),p(95),p(99),avg,min,max" +{ + echo "export BASE_DIR=${BASE_DIR}" + echo "export GOROOT=${GOROOT}" + echo "export GOPATH=${GOPATH}" + echo "export PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin" + echo "export XK6_BIN=${GOPATH}/bin/xk6" + echo "export K6_OUT=xk6-prometheus-rw" + echo "export K6_PROMETHEUS_RW_SERVER_URL=http://146.56.46.8:9090/api/v1/write" + echo "K6_PROMETHEUS_RW_TREND_STATS=\"p(90),p(95),p(99),avg,min,max\"" +} >> ~/.bashrc + +echo "📁 디렉토리 생성 및 이동: $BASE_DIR" +mkdir -p "$BASE_DIR" +cd "$BASE_DIR" + +echo "⬇️ Go $GO_VERSION 다운로드 중..." +curl -OL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" + +echo "📦 Go 압축 해제 중..." +tar -xzf "go${GO_VERSION}.linux-amd64.tar.gz" +rm "go${GO_VERSION}.linux-amd64.tar.gz" + +echo "✅ Go 버전 확인: $(go version)" + +echo "⬇️ xk6 설치 중..." +go install go.k6.io/xk6/cmd/xk6@latest + +echo "✅ xk6 설치 완료: $XK6_BIN" +$XK6_BIN --help > /dev/null && echo "✅ xk6 실행 가능" + +echo "⚙️ Prometheus remote-write 플러그인을 포함한 K6 빌드 시작" +$XK6_BIN build --with github.com/grafana/xk6-output-prometheus-remote@latest + +echo "✅ 빌드 완료: $(pwd)/k6" +ls -lh ./k6 + +echo "🎉 설치가 성공적으로 완료되었습니다!" diff --git a/config/load-test/k6/updatePost.json b/config/load-test/k6/updatePost.json new file mode 100644 index 0000000..660f0a8 --- /dev/null +++ b/config/load-test/k6/updatePost.json @@ -0,0 +1,5 @@ +{ + "postCategory": "자유", + "title": "수강신청 어떻게 하나요?", + "content": "수강신청 방법을 잘 알겠어요." +} diff --git a/config/load-test/k6/whole-user-flow.js b/config/load-test/k6/whole-user-flow.js new file mode 100644 index 0000000..6fc1b47 --- /dev/null +++ b/config/load-test/k6/whole-user-flow.js @@ -0,0 +1,372 @@ +import http from 'k6/http'; +import { sleep, check, fail } from 'k6'; + +// KST +const now = new Date(); +const kstOffset = 9 * 60; // 분 단위 +const d = new Date(now.getTime() + kstOffset * 60 * 1000); +const kst = d.toISOString().slice(0, 16); // "yyyy-mm-ddTHH:MM" + +// "mm/dd HH:MM" +const time = (() => { + const [yyyy, mm, dd, hh, min] = kst.split(/[-T:]/); + return `${mm}/${dd} ${hh}:${min}`; +})(); + +const BASE_URL = 'https://api.stage.solid-connection.com'; +const testId = 'whole-user-flow'; + +export const options = { + scenarios: { + user_flow: { + executor: 'per-vu-iterations', // VU별 반복 + vus: 10, // VU + iterations: 10, // VU 한 명당 실행할 횟수 + maxDuration: '15m', // 여유로 잡아 두기 + }, + }, + tags: { + testid: testId, + time: time, + }, +}; + +function authHeadersWithTags(token) { + return { + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json; charset=utf-8', + }, + tags: { + ...options.tags, + time: time, + }, + }; +} + +function login() { + // __VU: 현재 VU 인덱스 + const email = `user${__VU}@example.com`; + const password = 'password'; + + const res = http.post(`${BASE_URL}/auth/email/sign-in`, JSON.stringify({ + email: email, + password: password, + }), { + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + tags: { + name: '/auth/email/sign-in', + } + }); + if (res.status !== 200) { + fail('로그인 실패'); + } + return res.json('accessToken'); +} + +// universites +function getRecommendedUniversities(auth) { + http.get(`${BASE_URL}/universities/recommend`, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/recommend', + }, +}); +} +function likeUniversity(id, auth) { + http.post(`${BASE_URL}/universities/${id}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/{id}/like', + }, + }); +} +function isLikedUniversity(id, auth) { + http.get(`${BASE_URL}/universities/${id}/like`, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/{id}/like', + }, + }); +} +function getLikedUniversities(auth) { + http.get(`${BASE_URL}/universities/like`, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/like', + }, +}); +} +function cancelLikeUniversity(id, auth) { + http.del(`${BASE_URL}/universities/${id}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/{id}/like', + }, + }); +} +function searchUniversities(params) { + return http.get(`${BASE_URL}/universities/search?${params}`, { + tags: { + name: '/universities/search?{params}', + }, +}); +} +function getDetailedUniversityInfo(id) { + http.get(`${BASE_URL}/universities/${id}`, { + tags: { + name: '/universities/{id}', + }, + }); +} + +// my +function getMyInfo(auth) { + http.get(`${BASE_URL}/my`, { + ...auth, + tags: { + ...auth.tags, + name: '/my', + }, +}); +} + +// users +function checkNicknameExists(nickname) { + http.get(`${BASE_URL}/users/exists?nickname=${nickname}`, { + tags: { + name: '/users/exists?nickname={nickname}', + }, + }); +} + +// boards +function getBoards(auth) { + http.get(`${BASE_URL}/boards`, { + ...auth, + tags: { + ...auth.tags, + name: '/boards', + }, +}); +} +function getPostsByBoard(boardCode, auth) { + http.get(`${BASE_URL}/boards/${boardCode}`, { + ...auth, + tags: { + ...auth.tags, + name: '/boards/{boardCode}', + }, + }); +} + +// posts +const createPostJson = open('./createPost.json', 'b'); +function createPost(token) { + const formData = { + postCreateRequest: http.file(createPostJson, 'post.json', 'application/json'), + }; + const res = http.post(`${BASE_URL}/posts`, formData, { + headers: { + Authorization: `Bearer ${token}` + }, + tags: { + testid: testId, + time: time, + name: '/posts' + }, + }); + return res.json('id'); +} +const updatePostJson = open('./updatePost.json', 'b'); +function updatePost(postId, token) { + const formData = { + postUpdateRequest: http.file(updatePostJson, 'post.json', 'application/json'), + }; + http.patch(`${BASE_URL}/posts/${postId}`, formData, { + headers: { + Authorization: `Bearer ${token}` + }, + tags: { + testid: testId, + time: time, + name: '/posts/{postId}' + }, + }); +} +function getPostDetail(postId, auth) { + http.get(`${BASE_URL}/posts/${postId}`, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}', + }, + }); +} +function likePost(postId, auth) { + http.post(`${BASE_URL}/posts/${postId}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}/like', + }, + }); +} +function cancelLikePost(postId, auth) { + http.del(`${BASE_URL}/posts/${postId}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}/like', + }, + }); +} +function deletePost(postId, auth) { + http.del(`${BASE_URL}/posts/${postId}`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}', + }, + }); +} + +// comments +function createComment(postId, auth) { + const res = http.post( + `${BASE_URL}/comments`, + JSON.stringify({ postId, content: '댓글', parentId: null }), + { + ...auth, + tags: { + ...auth.tags, + name: '/comments', + }, + }); + return res.json('id'); +} +function updateComment(commentId, auth) { + http.patch( + `${BASE_URL}/comments/${commentId}`, + JSON.stringify({ content: '댓글 수정' }), + { + ...auth, + tags: { + ...auth.tags, + name: '/comments/{commentId}', + }, + } + ); +} +function deleteComment(commentId, auth) { + http.del(`${BASE_URL}/comments/${commentId}`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/comments/{commentId}', + }, + }); +} + +// scores +function getLanguageTests(auth) { + return http.get(`${BASE_URL}/scores/language-tests`, { + ...auth, + tags: { + ...auth.tags, + name: '/scores/language-tests', + }, + }); +} +function getGPAs(auth) { + return http.get(`${BASE_URL}/scores/gpas`, { + ...auth, + tags: { + ...auth.tags, + name: '/scores/gpas', + }, + }); +} + +// applications +function apply(gpaScoreId, languageTestScoreId, universityId, auth) { + http.post(`${BASE_URL}/applications`, JSON.stringify({ + gpaScoreId: gpaScoreId, + languageTestScoreId: languageTestScoreId, + universityChoiceRequest: { + firstChoiceUniversityId: universityId, + secondChoiceUniversityId: null, + thirdChoiceUniversityId: null + }, + }), { + ...auth, + tags: { + ...auth.tags, + name: '/applications', + }, + }); +} + +function getCompetitors(auth) { + http.get(`${BASE_URL}/applications/competitors`, { + ...auth, + tags: { + ...auth.tags, + name: '/applications/competitors', + }, + }); +} + +export default function () { + checkNicknameExists(encodeURIComponent('닉네임')); + const token = login(); + const auth = authHeadersWithTags(token); + + + getRecommendedUniversities(auth); + + const uniSearchRes = searchUniversities(''); // 이번학기 열린 대학 중 랜덤하게 id 가져오기 + const uniList = uniSearchRes.json(); + const universityId = uniList[Math.floor(Math.random() * uniList.length)].id; + + likeUniversity(universityId, auth); + isLikedUniversity(universityId, auth); + getLikedUniversities(auth); + cancelLikeUniversity(universityId, auth); + getDetailedUniversityInfo(universityId); + + getMyInfo(auth); + + getBoards(auth); + getPostsByBoard('FREE', auth); + + const postId = createPost(token); + updatePost(postId, token); + getPostDetail(postId, auth); + likePost(postId, auth); + cancelLikePost(postId, auth); + + const commentId = createComment(postId, auth); + updateComment(commentId, auth); + deleteComment(commentId, auth); + + deletePost(postId, auth); + + const langRes = getLanguageTests(auth); + const langList = langRes.json().languageTestScoreStatusResponseList; + const languageTestScoreId = langList[0].id; + + const gpaRes = getGPAs(auth); + const gpaList = gpaRes.json().gpaScoreStatusResponseList; + const gpaScoreId = gpaList[0].id; + + apply(gpaScoreId, languageTestScoreId, universityId, auth); + getCompetitors(auth); + + sleep(1); +} diff --git a/environment/stage/main.tf b/environment/stage/main.tf index 3f3e129..8e60bb5 100644 --- a/environment/stage/main.tf +++ b/environment/stage/main.tf @@ -6,8 +6,8 @@ data "aws_vpc" "default" { module "stage_stack" { source = "../../modules/app_stack" - env_name = "stage" - vpc_id = data.aws_vpc.default.id + env_name = "stage" + vpc_id = data.aws_vpc.default.id ami_id = var.ami_id @@ -15,13 +15,13 @@ module "stage_stack" { ec2_iam_instance_profile = var.ec2_iam_instance_profile # 키페어 및 접속 허용 - key_name = var.key_name + key_name = var.key_name # 인스턴스 스펙 - instance_type = var.server_instance_type + instance_type = var.server_instance_type # RDS 미사용 (Docker container로 대체) - enable_rds = false + enable_rds = false # 보안 그룹 규칙 api_ingress_rules = var.api_ingress_rules @@ -41,4 +41,6 @@ module "stage_stack" { redis_version = var.redis_version redis_exporter_version = var.redis_exporter_version alloy_version = var.alloy_version + + enable_k6_files = true } diff --git a/modules/app_stack/ec2.tf b/modules/app_stack/ec2.tf index b49aa52..e1da678 100644 --- a/modules/app_stack/ec2.tf +++ b/modules/app_stack/ec2.tf @@ -21,6 +21,57 @@ data "cloudinit_config" "app_init" { content = file("${path.module}/../common/scripts/docker_setup.sh") filename = "1_docker_install.sh" } + + dynamic "part" { + for_each = var.enable_k6_files ? [1] : [] + + content { + content_type = "text/cloud-config" + content = yamlencode({ + bootcmd = [ + "mkdir -p ${var.k6_install_dir}/script" + ] + write_files = [ + { + path = "${var.k6_install_dir}/createPost.json" + owner = "ubuntu:ubuntu" + permissions = "0644" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/createPost.json") + }, + { + path = "${var.k6_install_dir}/updatePost.json" + owner = "ubuntu:ubuntu" + permissions = "0644" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/updatePost.json") + }, + { + path = "${var.k6_install_dir}/whole-user-flow.js" + owner = "ubuntu:ubuntu" + permissions = "0644" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/whole-user-flow.js") + }, + { + path = "${var.k6_install_dir}/set_up_xk6.sh" + owner = "ubuntu:ubuntu" + permissions = "0755" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/set_up_xk6.sh") + }, + { + path = "${var.k6_install_dir}/script/set-load-test.sh" + owner = "ubuntu:ubuntu" + permissions = "0755" + encoding = "b64" + content = filebase64("${path.module}/../../config/load-test/k6/script/set-load-test.sh") + } + ] + }) + filename = "2_k6_files.yml" + } + } } # API Server (EC2) @@ -100,9 +151,9 @@ resource "null_resource" "update_side_infra" { triggers = { script_hash = sha256(templatefile("${path.module}/scripts/side_infra_setup.sh.tftpl", { - work_dir = var.work_dir - alloy_env_name = var.alloy_env_name - alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { + work_dir = var.work_dir + alloy_env_name = var.alloy_env_name + alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { loki_ip = data.aws_instance.monitoring_server.private_ip }) redis_version = var.redis_version @@ -120,9 +171,9 @@ resource "null_resource" "update_side_infra" { provisioner "file" { content = templatefile("${path.module}/scripts/side_infra_setup.sh.tftpl", { - work_dir = var.work_dir - alloy_env_name = var.alloy_env_name - alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { + work_dir = var.work_dir + alloy_env_name = var.alloy_env_name + alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { loki_ip = data.aws_instance.monitoring_server.private_ip }) redis_version = var.redis_version diff --git a/modules/app_stack/variables.tf b/modules/app_stack/variables.tf index 33f8b1a..c1d199c 100644 --- a/modules/app_stack/variables.tf +++ b/modules/app_stack/variables.tf @@ -67,7 +67,7 @@ variable "additional_db_users" { database = string privileges = list(string) })) - default = {} + default = {} } variable "db_engine_version" { @@ -155,3 +155,15 @@ variable "alloy_version" { description = "Docker image tag for Grafana Alloy" type = string } + +variable "enable_k6_files" { + description = "Whether to place k6 load test files on the API server during cloud-init" + type = bool + default = false +} + +variable "k6_install_dir" { + description = "Directory where k6 load test files are placed" + type = string + default = "/home/ubuntu/solid-connection-load-test/k6" +} diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 0283669..f929c01 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -1,61 +1,74 @@ -# Load Test Automation +# 부하 테스트 자동화 -This automation creates a temporary load test RDS instance, copies prod RDS data -into it, writes load test datasource values to Parameter Store, and switches the -stage application through SSM Run Command. +부하 테스트용 임시 RDS를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 +loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작과 종료는 GitHub +Actions에서 수동으로 실행합니다. -## GitHub Actions Flow +## 시작 -### Start +1. GitHub에서 **Actions > Load Test Start**를 엽니다. +2. **Run workflow**를 클릭합니다. +3. 기본값 그대로 실행합니다. -1. Open **Actions > Load Test Start**. -2. Click **Run workflow**. -3. Keep `switch_stage_to_loadtest` enabled to restart stage with - `dev,loadtest` profiles. -4. Keep `copy_prod_data` enabled to copy prod RDS data into the load test RDS. +입력값: -The workflow runs `scripts/load_test/start.sh`. +- `switch_stage_to_loadtest`: stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +- `copy_prod_data`: prod RDS 데이터를 loadtest RDS로 복사합니다. -### Stop +시작 workflow는 `scripts/load_test/start.sh`를 실행합니다. -1. Open **Actions > Load Test Stop**. -2. Click **Run workflow**. -3. Keep `restore_stage_dev` enabled to restart stage with the normal dev - compose configuration. -4. Keep `destroy_rds` enabled to destroy the load test Terraform stack. +## 종료 -The workflow runs `scripts/load_test/stop.sh`. +1. GitHub에서 **Actions > Load Test Stop**을 엽니다. +2. **Run workflow**를 클릭합니다. +3. 기본값 그대로 실행합니다. -## What Start Does +입력값: -1. Runs `terraform apply` in `environment/load_test`. -2. Creates the load test RDS and writes: +- `restore_stage_dev`: stage 앱을 기존 dev compose 구성으로 되돌립니다. +- `destroy_rds`: loadtest Terraform stack을 제거합니다. + +종료 workflow는 `scripts/load_test/stop.sh`를 실행합니다. + +## 시작 시 수행 작업 + +1. `environment/load_test`에서 `terraform apply`를 실행합니다. +2. loadtest RDS를 생성하고 아래 Parameter Store 값을 작성합니다. - `/solid-connection/loadtest/spring.datasource.url` - `/solid-connection/loadtest/spring.datasource.username` - `/solid-connection/loadtest/spring.datasource.password` -3. Switches the stage app to `dev,loadtest` profiles through SSM Run Command. -4. Stores DB migration credentials in temporary SSM parameters. -5. Runs `mysqldump` on the prod EC2 instance through SSM Run Command and - restores the dump into the load test RDS. -6. Deletes the temporary migration parameters. - -## What Stop Does - -1. Restores the stage app to the normal dev compose configuration through SSM - Run Command. -2. Runs `terraform destroy` for the load test Terraform stack. - -## Notes - -- GitHub Actions uses `AWS_ROLE_ARN` through OIDC. -- Private submodule checkout uses `GH_PAT`. -- No SSH private key is required for load test start/stop. -- The prod and stage EC2 instances are looked up by their `Name` tags. -- Prod DB username/password are read from Parameter Store. The default paths are - `/solid-connection/prod/spring.datasource.username` and - `/solid-connection/prod/spring.datasource.password`. -- Load test DB username/password are read from Parameter Store. The default paths - are `/solid-connection/loadtest/spring.datasource.username` and - `/solid-connection/loadtest/spring.datasource.password`. -- The load test RDS security group allows MySQL only from the security groups - attached to the prod and stage API EC2 instances. +3. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +4. migration용 임시 Parameter Store 값을 생성합니다. +5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS에 복원합니다. +6. migration용 임시 Parameter Store 값을 삭제합니다. + +## 종료 시 수행 작업 + +1. SSM RunCommand로 stage 앱을 기존 dev compose 구성으로 되돌립니다. +2. `environment/load_test`에서 `terraform destroy`를 실행합니다. + +## k6 파일 + +stage EC2를 새로 생성하는 경우 Terraform cloud-init이 +`/home/ubuntu/solid-connection-load-test/k6`에 k6 파일을 배치합니다. + +현재 포함된 파일: + +- `createPost.json` +- `updatePost.json` +- `whole-user-flow.js` +- `set_up_xk6.sh` +- `script/set-load-test.sh` + +기존 stage EC2는 재생성하지 않으므로 이 cloud-init 변경이 즉시 반영되지는 않습니다. + +## 참고 사항 + +- GitHub Actions는 OIDC로 `AWS_ROLE_ARN`을 assume합니다. +- private submodule checkout에는 `GH_PAT`를 사용합니다. +- SSH private key는 사용하지 않습니다. +- prod/stage EC2는 `Name` tag로 조회합니다. +- prod DB username/password는 Parameter Store에서 읽습니다. +- loadtest DB username/password도 Parameter Store에서 읽습니다. +- loadtest RDS security group은 prod/stage API EC2 security group에서 오는 MySQL + 접근만 허용합니다. From 7d59011e68aa77b0197d006a5b3dc1c50dc62ce9 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 14:33:31 +0900 Subject: [PATCH 06/11] =?UTF-8?q?fix:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=9E=90=EB=8F=99=ED=99=94=20=EB=A6=AC?= =?UTF-8?q?=EB=B7=B0=20=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: load_test Terraform plan workflow를 추가했습니다. - 상세내용: loadtest RDS 네트워크를 stage EC2 VPC 기준으로 생성하도록 수정했습니다. - 상세내용: SSM 명령 timeout, dump cleanup, k6 파일 동기화, 데이터 복사 후 stage 전환 순서를 반영했습니다. - 상세내용: k6 설정과 응답 검증 오류를 수정했습니다. --- .github/workflows/terraform-plan.yml | 70 ++++++++++++- config/load-test/k6/set_up_xk6.sh | 30 +++--- config/load-test/k6/whole-user-flow.js | 28 +++-- environment/load_test/main.tf | 26 ++--- environment/load_test/output.tf | 20 ++++ environment/load_test/variables.tf | 12 +++ scripts/load_test/README.md | 69 ++++++------ scripts/load_test/start.sh | 140 +++++++++++++++++++------ scripts/load_test/stop.sh | 14 +++ 9 files changed, 309 insertions(+), 100 deletions(-) diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml index 870f6e3..5f5d1af 100644 --- a/.github/workflows/terraform-plan.yml +++ b/.github/workflows/terraform-plan.yml @@ -21,6 +21,7 @@ jobs: global: ${{ steps.filter.outputs.global }} prod: ${{ steps.filter.outputs.prod }} stage: ${{ steps.filter.outputs.stage }} + load_test: ${{ steps.filter.outputs.load_test }} monitoring: ${{ steps.filter.outputs.monitoring }} steps: - uses: actions/checkout@v4 @@ -49,6 +50,9 @@ jobs: - 'modules/common/**' - 'config/secrets/stage.tfvars' - 'config/secrets/app_stack.tfvars' + load_test: + - 'environment/load_test/**' + - 'config/secrets/load_test.tfvars' monitoring: - 'environment/monitoring/**' - 'modules/monitoring_stack/**' @@ -373,6 +377,69 @@ jobs: if: steps.plan.outputs.exitcode == '1' run: exit 1 + plan-load_test: + needs: detect-changes + if: needs.detect-changes.outputs.load_test == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_PLAN_ROLE_ARN }} + aws-region: ap-northeast-2 + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + - name: Terraform Init + working-directory: environment/load_test + run: terraform init + - name: Terraform Plan + id: plan + working-directory: environment/load_test + run: | + terraform plan -no-color \ + -var-file="../../config/secrets/load_test.tfvars" \ + 2>&1 | tee plan_output.txt + echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT + - name: Upload Plan Artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: terraform-plan-load-test + path: environment/load_test/plan_output.txt + - name: Post Plan Comment + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const marker = ''; + const planFile = 'environment/load_test/plan_output.txt'; + const output = fs.existsSync(planFile) ? fs.readFileSync(planFile, 'utf8') : ''; + const summary = output.split('\n').find(l => /^Plan:/.test(l)) || output.split('\n').find(l => /No changes/.test(l)) || (output ? '(plan result parse failed)' : 'plan failed before writing output. Check workflow logs.'); + const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = `${marker}\n## Terraform Plan: \`load_test\`\n\n${summary}\n\n> Full plan output is kept in the workflow artifact for security. Check [workflow run artifact](${runUrl}).`; + + const { data: comments } = await github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + }); + const existing = comments.find(c => c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ comment_id: existing.id, owner: context.repo.owner, repo: context.repo.repo, body }); + } else { + await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body }); + } + - name: Plan Status Check + if: steps.plan.outputs.exitcode == '1' + run: exit 1 + plan-monitoring: needs: detect-changes if: needs.detect-changes.outputs.monitoring == 'true' @@ -438,7 +505,7 @@ jobs: run: exit 1 trigger-coderabbit: - needs: [plan-bootstrap, plan-global, plan-prod, plan-stage, plan-monitoring] + needs: [plan-bootstrap, plan-global, plan-prod, plan-stage, plan-load_test, plan-monitoring] if: | always() && ( @@ -446,6 +513,7 @@ jobs: needs.plan-global.result == 'success' || needs.plan-global.result == 'failure' || needs.plan-prod.result == 'success' || needs.plan-prod.result == 'failure' || needs.plan-stage.result == 'success' || needs.plan-stage.result == 'failure' || + needs.plan-load_test.result == 'success' || needs.plan-load_test.result == 'failure' || needs.plan-monitoring.result == 'success' || needs.plan-monitoring.result == 'failure' ) runs-on: ubuntu-latest diff --git a/config/load-test/k6/set_up_xk6.sh b/config/load-test/k6/set_up_xk6.sh index 005fb3d..01ad6d6 100644 --- a/config/load-test/k6/set_up_xk6.sh +++ b/config/load-test/k6/set_up_xk6.sh @@ -2,8 +2,7 @@ set -euo pipefail -# 에러 발생 시 메시지 출력 -trap 'echo "❌ 오류 발생! 스크립트 실행이 중단되었습니다." >&2' ERR +trap 'echo "xk6 setup failed" >&2' ERR export GO_VERSION=1.22.2 export BASE_DIR=/home/ubuntu/solid-connection-load-test/k6 @@ -14,6 +13,7 @@ export XK6_BIN=${GOPATH}/bin/xk6 export K6_OUT=xk6-prometheus-rw export K6_PROMETHEUS_RW_SERVER_URL=http://132.145.83.182:9090/api/v1/write export K6_PROMETHEUS_RW_TREND_STATS="p(90),p(95),p(99),avg,min,max" + { echo "export BASE_DIR=${BASE_DIR}" echo "export GOROOT=${GOROOT}" @@ -21,33 +21,33 @@ export K6_PROMETHEUS_RW_TREND_STATS="p(90),p(95),p(99),avg,min,max" echo "export PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin" echo "export XK6_BIN=${GOPATH}/bin/xk6" echo "export K6_OUT=xk6-prometheus-rw" - echo "export K6_PROMETHEUS_RW_SERVER_URL=http://146.56.46.8:9090/api/v1/write" - echo "K6_PROMETHEUS_RW_TREND_STATS=\"p(90),p(95),p(99),avg,min,max\"" + echo "export K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL}" + echo "export K6_PROMETHEUS_RW_TREND_STATS=\"${K6_PROMETHEUS_RW_TREND_STATS}\"" } >> ~/.bashrc -echo "📁 디렉토리 생성 및 이동: $BASE_DIR" +echo "Create and enter ${BASE_DIR}" mkdir -p "$BASE_DIR" cd "$BASE_DIR" -echo "⬇️ Go $GO_VERSION 다운로드 중..." +echo "Download Go ${GO_VERSION}" curl -OL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" -echo "📦 Go 압축 해제 중..." +echo "Extract Go" tar -xzf "go${GO_VERSION}.linux-amd64.tar.gz" rm "go${GO_VERSION}.linux-amd64.tar.gz" -echo "✅ Go 버전 확인: $(go version)" +echo "Go version: $(go version)" -echo "⬇️ xk6 설치 중..." +echo "Install xk6" go install go.k6.io/xk6/cmd/xk6@latest -echo "✅ xk6 설치 완료: $XK6_BIN" -$XK6_BIN --help > /dev/null && echo "✅ xk6 실행 가능" +echo "xk6 installed: ${XK6_BIN}" +"$XK6_BIN" --help > /dev/null && echo "xk6 executable is available" -echo "⚙️ Prometheus remote-write 플러그인을 포함한 K6 빌드 시작" -$XK6_BIN build --with github.com/grafana/xk6-output-prometheus-remote@latest +echo "Build k6 with Prometheus remote-write output" +"$XK6_BIN" build --with github.com/grafana/xk6-output-prometheus-remote@latest -echo "✅ 빌드 완료: $(pwd)/k6" +echo "Build complete: $(pwd)/k6" ls -lh ./k6 -echo "🎉 설치가 성공적으로 완료되었습니다!" +echo "xk6 setup completed" diff --git a/config/load-test/k6/whole-user-flow.js b/config/load-test/k6/whole-user-flow.js index 6fc1b47..a40f82c 100644 --- a/config/load-test/k6/whole-user-flow.js +++ b/config/load-test/k6/whole-user-flow.js @@ -293,6 +293,20 @@ function getGPAs(auth) { }); } +function requireArray(value, name) { + if (!Array.isArray(value) || value.length === 0) { + fail(`${name} response is empty or invalid`); + } + return value; +} + +function requireId(value, name) { + if (!value || value.id === undefined || value.id === null) { + fail(`${name} response does not contain id`); + } + return value.id; +} + // applications function apply(gpaScoreId, languageTestScoreId, universityId, auth) { http.post(`${BASE_URL}/applications`, JSON.stringify({ @@ -323,7 +337,7 @@ function getCompetitors(auth) { } export default function () { - checkNicknameExists(encodeURIComponent('닉네임')); + checkNicknameExists(encodeURIComponent('loadtest-user')); const token = login(); const auth = authHeadersWithTags(token); @@ -331,8 +345,8 @@ export default function () { getRecommendedUniversities(auth); const uniSearchRes = searchUniversities(''); // 이번학기 열린 대학 중 랜덤하게 id 가져오기 - const uniList = uniSearchRes.json(); - const universityId = uniList[Math.floor(Math.random() * uniList.length)].id; + const uniList = requireArray(uniSearchRes.json(), 'universities/search'); + const universityId = requireId(uniList[Math.floor(Math.random() * uniList.length)], 'universities/search item'); likeUniversity(universityId, auth); isLikedUniversity(universityId, auth); @@ -358,12 +372,12 @@ export default function () { deletePost(postId, auth); const langRes = getLanguageTests(auth); - const langList = langRes.json().languageTestScoreStatusResponseList; - const languageTestScoreId = langList[0].id; + const langList = requireArray(langRes.json().languageTestScoreStatusResponseList, 'scores/language-tests'); + const languageTestScoreId = requireId(langList[0], 'scores/language-tests item'); const gpaRes = getGPAs(auth); - const gpaList = gpaRes.json().gpaScoreStatusResponseList; - const gpaScoreId = gpaList[0].id; + const gpaList = requireArray(gpaRes.json().gpaScoreStatusResponseList, 'scores/gpas'); + const gpaScoreId = requireId(gpaList[0], 'scores/gpas item'); apply(gpaScoreId, languageTestScoreId, universityId, auth); getCompetitors(auth); diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index 72b1c58..2b5716f 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -1,14 +1,3 @@ -data "aws_vpc" "default" { - default = true -} - -data "aws_subnets" "default" { - filter { - name = "vpc-id" - values = [data.aws_vpc.default.id] - } -} - data "aws_instance" "prod_api" { filter { name = "tag:Name" @@ -33,6 +22,17 @@ data "aws_instance" "stage_api" { } } +data "aws_subnet" "stage_api" { + id = data.aws_instance.stage_api.subnet_id +} + +data "aws_subnets" "target" { + filter { + name = "vpc-id" + values = [data.aws_subnet.stage_api.vpc_id] + } +} + data "aws_db_instance" "prod" { db_instance_identifier = var.prod_rds_identifier } @@ -59,7 +59,7 @@ locals { resource "aws_security_group" "load_test_db" { name = "sc-load-test-db-sg" description = "Security group for load test RDS" - vpc_id = data.aws_vpc.default.id + vpc_id = data.aws_subnet.stage_api.vpc_id egress { from_port = 0 @@ -87,7 +87,7 @@ resource "aws_security_group_rule" "load_test_db_mysql" { resource "aws_db_subnet_group" "load_test" { name = "sc-load-test-db-subnet-group" - subnet_ids = data.aws_subnets.default.ids + subnet_ids = data.aws_subnets.target.ids tags = { Name = "solid-connection-load-test-db-subnet-group" diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf index 55390ac..ee0bc25 100644 --- a/environment/load_test/output.tf +++ b/environment/load_test/output.tf @@ -47,3 +47,23 @@ output "load_test_ssm_parameter_prefix" { description = "SSM Parameter Store prefix for load test datasource values" value = var.load_test_parameter_prefix } + +output "load_test_db_username_parameter_name" { + description = "SSM parameter name containing the load test DB username" + value = var.load_test_db_username_parameter_name +} + +output "load_test_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the load test DB password" + value = var.load_test_db_password_parameter_name +} + +output "prod_db_username_parameter_name" { + description = "SSM parameter name containing the prod DB username" + value = var.prod_db_username_parameter_name +} + +output "prod_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the prod DB password" + value = var.prod_db_password_parameter_name +} diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index 7d4d639..efdfa28 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -40,6 +40,18 @@ variable "load_test_db_password_parameter_name" { type = string } +variable "prod_db_username_parameter_name" { + description = "SSM parameter name containing the prod DB username" + type = string + default = "/solid-connection/prod/spring.datasource.username" +} + +variable "prod_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the prod DB password" + type = string + default = "/solid-connection/prod/spring.datasource.password" +} + variable "kms_key_arn" { description = "KMS key ARN for RDS storage encryption" type = string diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index f929c01..3647c44 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -1,58 +1,56 @@ # 부하 테스트 자동화 -부하 테스트용 임시 RDS를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 -loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작과 종료는 GitHub -Actions에서 수동으로 실행합니다. +부하 테스트용 임시 RDS를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작과 종료는 GitHub Actions에서 수동으로 실행합니다. + +## 원칙 + +- 사람이 로컬에서 `terraform apply` 또는 `terraform destroy`를 직접 실행하지 않습니다. +- 시작은 **Actions > Load Test Start** workflow로 실행합니다. +- 종료는 **Actions > Load Test Stop** workflow로 실행합니다. +- SSH private key는 사용하지 않습니다. stage/prod EC2 작업은 SSM RunCommand로 실행합니다. ## 시작 1. GitHub에서 **Actions > Load Test Start**를 엽니다. 2. **Run workflow**를 클릭합니다. -3. 기본값 그대로 실행합니다. +3. 필요한 입력값을 선택하고 실행합니다. 입력값: - `switch_stage_to_loadtest`: stage 앱을 `dev,loadtest` 프로필로 재기동합니다. - `copy_prod_data`: prod RDS 데이터를 loadtest RDS로 복사합니다. -시작 workflow는 `scripts/load_test/start.sh`를 실행합니다. +workflow는 GitHub Actions runner에서 `environment/load_test`의 Terraform을 apply하고 `scripts/load_test/start.sh`를 실행합니다. + +## 시작 시 수행 작업 + +1. GitHub Actions가 `environment/load_test`에서 Terraform apply를 실행합니다. +2. loadtest RDS와 보안 그룹을 생성합니다. RDS는 stage EC2가 속한 VPC/subnet 기준으로 생성됩니다. +3. loadtest datasource 값을 Parameter Store에 작성합니다. +4. prod DB와 loadtest DB 접속 정보를 Parameter Store에서 읽습니다. +5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS로 복원합니다. +6. stage EC2에 k6 파일을 `/home/ubuntu/solid-connection-load-test/k6` 경로로 동기화합니다. +7. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +8. 데이터 이관용 임시 Parameter Store 값을 삭제합니다. ## 종료 1. GitHub에서 **Actions > Load Test Stop**을 엽니다. 2. **Run workflow**를 클릭합니다. -3. 기본값 그대로 실행합니다. +3. 필요한 입력값을 선택하고 실행합니다. 입력값: - `restore_stage_dev`: stage 앱을 기존 dev compose 구성으로 되돌립니다. - `destroy_rds`: loadtest Terraform stack을 제거합니다. -종료 workflow는 `scripts/load_test/stop.sh`를 실행합니다. - -## 시작 시 수행 작업 - -1. `environment/load_test`에서 `terraform apply`를 실행합니다. -2. loadtest RDS를 생성하고 아래 Parameter Store 값을 작성합니다. - - `/solid-connection/loadtest/spring.datasource.url` - - `/solid-connection/loadtest/spring.datasource.username` - - `/solid-connection/loadtest/spring.datasource.password` -3. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. -4. migration용 임시 Parameter Store 값을 생성합니다. -5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS에 복원합니다. -6. migration용 임시 Parameter Store 값을 삭제합니다. - -## 종료 시 수행 작업 - -1. SSM RunCommand로 stage 앱을 기존 dev compose 구성으로 되돌립니다. -2. `environment/load_test`에서 `terraform destroy`를 실행합니다. +workflow는 `scripts/load_test/stop.sh`를 실행하고, 선택값에 따라 stage 복구와 Terraform destroy를 수행합니다. ## k6 파일 -stage EC2를 새로 생성하는 경우 Terraform cloud-init이 -`/home/ubuntu/solid-connection-load-test/k6`에 k6 파일을 배치합니다. +stage EC2를 새로 생성하는 경우 Terraform cloud-init이 k6 파일을 배치합니다. 기존 stage EC2는 cloud-init이 다시 실행되지 않으므로, **Load Test Start** workflow가 실행될 때 SSM으로 k6 파일을 다시 동기화합니다. -현재 포함된 파일: +포함 파일: - `createPost.json` - `updatePost.json` @@ -60,15 +58,18 @@ stage EC2를 새로 생성하는 경우 Terraform cloud-init이 - `set_up_xk6.sh` - `script/set-load-test.sh` -기존 stage EC2는 재생성하지 않으므로 이 cloud-init 변경이 즉시 반영되지는 않습니다. +stage EC2에 접속해 수동으로 실행해야 한다면 다음 경로에서 실행합니다. + +```bash +cd /home/ubuntu/solid-connection-load-test/k6 +./set_up_xk6.sh +./script/set-load-test.sh +``` -## 참고 사항 +## 참고 - GitHub Actions는 OIDC로 `AWS_ROLE_ARN`을 assume합니다. - private submodule checkout에는 `GH_PAT`를 사용합니다. -- SSH private key는 사용하지 않습니다. - prod/stage EC2는 `Name` tag로 조회합니다. -- prod DB username/password는 Parameter Store에서 읽습니다. -- loadtest DB username/password도 Parameter Store에서 읽습니다. -- loadtest RDS security group은 prod/stage API EC2 security group에서 오는 MySQL - 접근만 허용합니다. +- prod/loadtest DB username/password는 Parameter Store에서 읽습니다. +- loadtest RDS security group은 prod/stage API EC2 security group에서 오는 MySQL 접근만 허용합니다. diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh index 1f9c75a..21ab43d 100644 --- a/scripts/load_test/start.sh +++ b/scripts/load_test/start.sh @@ -3,15 +3,18 @@ set -euo pipefail TERRAFORM_DIR="environment/load_test" VAR_FILE="../../config/secrets/load_test.tfvars" -DATABASE_NAME="solid_connection" +DATABASE_NAME="" MIGRATION_PARAMETER_PREFIX="/solid-connection/loadtest/migration" -PROD_DB_USERNAME_PARAMETER="/solid-connection/prod/spring.datasource.username" -PROD_DB_PASSWORD_PARAMETER="/solid-connection/prod/spring.datasource.password" -LOADTEST_DB_USERNAME_PARAMETER="/solid-connection/loadtest/spring.datasource.username" -LOADTEST_DB_PASSWORD_PARAMETER="/solid-connection/loadtest/spring.datasource.password" +PROD_DB_USERNAME_PARAMETER="" +PROD_DB_PASSWORD_PARAMETER="" +LOADTEST_DB_USERNAME_PARAMETER="" +LOADTEST_DB_PASSWORD_PARAMETER="" SWITCH_STAGE_TO_LOADTEST="false" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" +STAGE_K6_DIR="/home/ubuntu/solid-connection-load-test/k6" +LOCAL_K6_DIR="config/load-test/k6" +SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-1800}" SKIP_TERRAFORM_APPLY="false" SKIP_DATA_COPY="false" @@ -22,15 +25,18 @@ Usage: scripts/load_test/start.sh [options] Options: --terraform-dir PATH Default: environment/load_test --var-file PATH Default: ../../config/secrets/load_test.tfvars - --prod-db-username-parameter Default: /solid-connection/prod/spring.datasource.username - --prod-db-password-parameter Default: /solid-connection/prod/spring.datasource.password - --loadtest-db-username-parameter Default: /solid-connection/loadtest/spring.datasource.username - --loadtest-db-password-parameter Default: /solid-connection/loadtest/spring.datasource.password - --database-name VALUE Default: solid_connection + --prod-db-username-parameter Default: Terraform output prod_db_username_parameter_name + --prod-db-password-parameter Default: Terraform output prod_db_password_parameter_name + --loadtest-db-username-parameter Default: Terraform output load_test_db_username_parameter_name + --loadtest-db-password-parameter Default: Terraform output load_test_db_password_parameter_name + --database-name VALUE Default: Terraform output load_test_db_name --migration-prefix VALUE Default: /solid-connection/loadtest/migration --switch-stage-to-loadtest Restart stage app through SSM with dev,loadtest profiles --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml + --stage-k6-dir PATH Default: /home/ubuntu/solid-connection-load-test/k6 + --local-k6-dir PATH Default: config/load-test/k6 + --ssm-command-timeout-seconds Default: 1800 --skip-terraform-apply --skip-data-copy -h, --help @@ -50,6 +56,9 @@ while [[ $# -gt 0 ]]; do --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --stage-k6-dir) STAGE_K6_DIR="$2"; shift 2 ;; + --local-k6-dir) LOCAL_K6_DIR="$2"; shift 2 ;; + --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; --skip-data-copy) SKIP_DATA_COPY="true"; shift ;; -h|--help) usage; exit 0 ;; @@ -76,6 +85,7 @@ require_command() { require_command terraform require_command aws require_command jq +require_command base64 tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" @@ -87,6 +97,7 @@ send_ssm_command() { local commands_json="$3" local command_id + local started_at command_id="$(aws ssm send-command \ --instance-ids "$instance_id" \ --document-name "AWS-RunShellScript" \ @@ -94,6 +105,7 @@ send_ssm_command() { --parameters "$commands_json" \ --query "Command.CommandId" \ --output text)" + started_at="$(date +%s)" local status while true; do @@ -104,6 +116,15 @@ send_ssm_command() { --query "Status" \ --output text 2>/dev/null || true)" + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command timed out after ${SSM_COMMAND_TIMEOUT_SECONDS}s: $comment" >&2 + exit 1 + fi + case "$status" in Pending|InProgress|Delayed|"") continue ;; Success) break ;; @@ -119,6 +140,51 @@ send_ssm_command() { done } +file_base64() { + base64 "$1" | tr -d '\n' +} + +sync_stage_k6_files() { + local instance_id="$1" + local commands + commands="$(jq -cn \ + --arg target_dir "$STAGE_K6_DIR" \ + '{ commands: ["set -euo pipefail", "mkdir -p \($target_dir)/script"] }')" + + local relative_path + for relative_path in \ + "createPost.json" \ + "updatePost.json" \ + "whole-user-flow.js" \ + "set_up_xk6.sh" \ + "script/set-load-test.sh"; do + local source_path="${LOCAL_K6_DIR}/${relative_path}" + if [[ ! -f "$source_path" ]]; then + echo "Missing k6 file: $source_path" >&2 + exit 1 + fi + + commands="$(jq -cn \ + --argjson current "$commands" \ + --arg target "${STAGE_K6_DIR}/${relative_path}" \ + --arg content "$(file_base64 "$source_path")" \ + '$current | .commands += [ + "mkdir -p \"$(dirname \"\($target)\")\"", + "printf %s \($content | @sh) | base64 -d > \($target | @sh)" + ]')" + done + + commands="$(jq -cn \ + --argjson current "$commands" \ + --arg target_dir "$STAGE_K6_DIR" \ + '$current | .commands += [ + "chmod +x \($target_dir)/set_up_xk6.sh \($target_dir)/script/set-load-test.sh", + "chown -R ubuntu:ubuntu \($target_dir)" + ]')" + + send_ssm_command "$instance_id" "Sync k6 files to stage EC2" "$commands" +} + delete_temp_parameters() { aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" >/dev/null 2>&1 || true aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" >/dev/null 2>&1 || true @@ -138,27 +204,17 @@ prod_endpoint="$(tf_output prod_rds_endpoint)" prod_port="$(tf_output prod_rds_port)" loadtest_endpoint="$(tf_output load_test_rds_endpoint)" loadtest_port="$(tf_output load_test_rds_port)" +loadtest_db_name="$(tf_output load_test_db_name)" +tf_prod_db_username_parameter="$(tf_output prod_db_username_parameter_name)" +tf_prod_db_password_parameter="$(tf_output prod_db_password_parameter_name)" +tf_loadtest_db_username_parameter="$(tf_output load_test_db_username_parameter_name)" +tf_loadtest_db_password_parameter="$(tf_output load_test_db_password_parameter_name)" -if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then - stage_commands_json="$(jq -cn \ - --arg app_dir "$STAGE_APP_DIR" \ - --arg compose_file "$STAGE_COMPOSE_FILE" \ - '{ - commands: [ - "set -euo pipefail", - "cd \($app_dir)", - "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", - "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", - "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", - "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", - "cat > docker-compose.loadtest.override.yml <<'\''YAML'\''\nservices:\n solid-connection-dev:\n environment:\n - SPRING_PROFILES_ACTIVE=dev,loadtest\n - AWS_REGION=ap-northeast-2\n - SPRING_DATA_REDIS_HOST=127.0.0.1\n - SPRING_DATA_REDIS_PORT=6379\nYAML", - "docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml down || true", - "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml up -d solid-connection-dev" - ] - }')" - - send_ssm_command "$stage_instance_id" "Switch stage app to load test datasource" "$stage_commands_json" -fi +DATABASE_NAME="${DATABASE_NAME:-$loadtest_db_name}" +PROD_DB_USERNAME_PARAMETER="${PROD_DB_USERNAME_PARAMETER:-$tf_prod_db_username_parameter}" +PROD_DB_PASSWORD_PARAMETER="${PROD_DB_PASSWORD_PARAMETER:-$tf_prod_db_password_parameter}" +LOADTEST_DB_USERNAME_PARAMETER="${LOADTEST_DB_USERNAME_PARAMETER:-$tf_loadtest_db_username_parameter}" +LOADTEST_DB_PASSWORD_PARAMETER="${LOADTEST_DB_PASSWORD_PARAMETER:-$tf_loadtest_db_password_parameter}" if [[ "$SKIP_DATA_COPY" != "true" ]]; then trap delete_temp_parameters EXIT @@ -226,6 +282,7 @@ if [[ "$SKIP_DATA_COPY" != "true" ]]; then "LOAD_USER=$(aws ssm get-parameter --name \($prefix)/loadtest-db-username --query Parameter.Value --output text)", "LOAD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/loadtest-db-password --with-decryption --query Parameter.Value --output text)", "DUMP_FILE=/tmp/solid-connection-loadtest-$(date +%Y%m%d%H%M%S).sql.gz", + "trap '\''rm -f \"$DUMP_FILE\"'\'' EXIT", "MYSQL_PWD=\"$PROD_PASSWORD\" mysqldump --single-transaction --set-gtid-purged=OFF --column-statistics=0 -h \($prod_endpoint) -P \($prod_port) -u \"$PROD_USER\" \($database) | gzip > \"$DUMP_FILE\"", "MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" -e \"DROP DATABASE IF EXISTS \\\`\($database)\\\`; CREATE DATABASE \\\`\($database)\\\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;\"", "gunzip -c \"$DUMP_FILE\" | MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" \($database)", @@ -236,6 +293,29 @@ if [[ "$SKIP_DATA_COPY" != "true" ]]; then send_ssm_command "$prod_instance_id" "Copy prod RDS data to load test RDS" "$copy_commands_json" fi +if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then + sync_stage_k6_files "$stage_instance_id" + + stage_commands_json="$(jq -cn \ + --arg app_dir "$STAGE_APP_DIR" \ + --arg compose_file "$STAGE_COMPOSE_FILE" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($app_dir)", + "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", + "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", + "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", + "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", + "cat > docker-compose.loadtest.override.yml <<'\''YAML'\''\nservices:\n solid-connection-dev:\n environment:\n - SPRING_PROFILES_ACTIVE=dev,loadtest\n - AWS_REGION=ap-northeast-2\n - SPRING_DATA_REDIS_HOST=127.0.0.1\n - SPRING_DATA_REDIS_PORT=6379\nYAML", + "docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml down || true", + "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml up -d solid-connection-dev" + ] + }')" + + send_ssm_command "$stage_instance_id" "Switch stage app to load test datasource" "$stage_commands_json" +fi + echo "Load test environment is ready." echo "RDS endpoint: ${loadtest_endpoint}:${loadtest_port}" echo "Stage instance: ${stage_instance_id}" diff --git a/scripts/load_test/stop.sh b/scripts/load_test/stop.sh index 62687fb..005a61c 100644 --- a/scripts/load_test/stop.sh +++ b/scripts/load_test/stop.sh @@ -6,6 +6,7 @@ VAR_FILE="../../config/secrets/load_test.tfvars" RESTORE_STAGE_DEV="false" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" +SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-900}" SKIP_TERRAFORM_DESTROY="false" usage() { @@ -18,6 +19,7 @@ Options: --restore-stage-dev Restart stage app through SSM with dev profile --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml + --ssm-command-timeout-seconds Default: 900 --skip-terraform-destroy -h, --help EOF @@ -30,6 +32,7 @@ while [[ $# -gt 0 ]]; do --restore-stage-dev) RESTORE_STAGE_DEV="true"; shift ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; --skip-terraform-destroy) SKIP_TERRAFORM_DESTROY="true"; shift ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1" >&2; usage; exit 1 ;; @@ -57,6 +60,7 @@ send_ssm_command() { local commands_json="$3" local command_id + local started_at command_id="$(aws ssm send-command \ --instance-ids "$instance_id" \ --document-name "AWS-RunShellScript" \ @@ -64,6 +68,7 @@ send_ssm_command() { --parameters "$commands_json" \ --query "Command.CommandId" \ --output text)" + started_at="$(date +%s)" local status while true; do @@ -74,6 +79,15 @@ send_ssm_command() { --query "Status" \ --output text 2>/dev/null || true)" + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command timed out after ${SSM_COMMAND_TIMEOUT_SECONDS}s: $comment" >&2 + exit 1 + fi + case "$status" in Pending|InProgress|Delayed|"") continue ;; Success) break ;; From 4878107fd7e813b788e4baa60703ce8466e8b8ad Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:04:55 +0900 Subject: [PATCH 07/11] =?UTF-8?q?chore:=20=EB=B6=80=ED=95=98=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20plan=20=EC=9E=90=EB=8F=99=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20=EC=A0=9C=EC=99=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: 임시로 생성되는 load_test 환경을 PR Terraform plan 대상에서 제외했습니다. - 상세내용: load_test apply와 destroy는 수동 GitHub Actions workflow에서만 실행하도록 정리했습니다. --- .github/workflows/terraform-plan.yml | 70 +--------------------------- 1 file changed, 1 insertion(+), 69 deletions(-) diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml index 5f5d1af..870f6e3 100644 --- a/.github/workflows/terraform-plan.yml +++ b/.github/workflows/terraform-plan.yml @@ -21,7 +21,6 @@ jobs: global: ${{ steps.filter.outputs.global }} prod: ${{ steps.filter.outputs.prod }} stage: ${{ steps.filter.outputs.stage }} - load_test: ${{ steps.filter.outputs.load_test }} monitoring: ${{ steps.filter.outputs.monitoring }} steps: - uses: actions/checkout@v4 @@ -50,9 +49,6 @@ jobs: - 'modules/common/**' - 'config/secrets/stage.tfvars' - 'config/secrets/app_stack.tfvars' - load_test: - - 'environment/load_test/**' - - 'config/secrets/load_test.tfvars' monitoring: - 'environment/monitoring/**' - 'modules/monitoring_stack/**' @@ -377,69 +373,6 @@ jobs: if: steps.plan.outputs.exitcode == '1' run: exit 1 - plan-load_test: - needs: detect-changes - if: needs.detect-changes.outputs.load_test == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - token: ${{ secrets.GH_PAT }} - persist-credentials: false - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_PLAN_ROLE_ARN }} - aws-region: ap-northeast-2 - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: ${{ env.TF_VERSION }} - terraform_wrapper: false - - name: Terraform Init - working-directory: environment/load_test - run: terraform init - - name: Terraform Plan - id: plan - working-directory: environment/load_test - run: | - terraform plan -no-color \ - -var-file="../../config/secrets/load_test.tfvars" \ - 2>&1 | tee plan_output.txt - echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - - name: Upload Plan Artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: terraform-plan-load-test - path: environment/load_test/plan_output.txt - - name: Post Plan Comment - if: always() - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const marker = ''; - const planFile = 'environment/load_test/plan_output.txt'; - const output = fs.existsSync(planFile) ? fs.readFileSync(planFile, 'utf8') : ''; - const summary = output.split('\n').find(l => /^Plan:/.test(l)) || output.split('\n').find(l => /No changes/.test(l)) || (output ? '(plan result parse failed)' : 'plan failed before writing output. Check workflow logs.'); - const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const body = `${marker}\n## Terraform Plan: \`load_test\`\n\n${summary}\n\n> Full plan output is kept in the workflow artifact for security. Check [workflow run artifact](${runUrl}).`; - - const { data: comments } = await github.rest.issues.listComments({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - }); - const existing = comments.find(c => c.body.includes(marker)); - if (existing) { - await github.rest.issues.updateComment({ comment_id: existing.id, owner: context.repo.owner, repo: context.repo.repo, body }); - } else { - await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body }); - } - - name: Plan Status Check - if: steps.plan.outputs.exitcode == '1' - run: exit 1 - plan-monitoring: needs: detect-changes if: needs.detect-changes.outputs.monitoring == 'true' @@ -505,7 +438,7 @@ jobs: run: exit 1 trigger-coderabbit: - needs: [plan-bootstrap, plan-global, plan-prod, plan-stage, plan-load_test, plan-monitoring] + needs: [plan-bootstrap, plan-global, plan-prod, plan-stage, plan-monitoring] if: | always() && ( @@ -513,7 +446,6 @@ jobs: needs.plan-global.result == 'success' || needs.plan-global.result == 'failure' || needs.plan-prod.result == 'success' || needs.plan-prod.result == 'failure' || needs.plan-stage.result == 'success' || needs.plan-stage.result == 'failure' || - needs.plan-load_test.result == 'success' || needs.plan-load_test.result == 'failure' || needs.plan-monitoring.result == 'success' || needs.plan-monitoring.result == 'failure' ) runs-on: ubuntu-latest From 4ebeb6ceaca6fbd857f4d8a934116979e6f1913c Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:05:04 +0900 Subject: [PATCH 08/11] =?UTF-8?q?feat:=20k6=20=EC=A0=84=EC=9A=A9=20?= =?UTF-8?q?=EB=B6=80=ED=95=98=20=EC=83=9D=EC=84=B1=20EC2=20=EA=B5=AC?= =?UTF-8?q?=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: load_test Terraform에 k6 전용 EC2와 보안 그룹을 추가했습니다. - 상세내용: stage EC2에는 k6 파일을 배치하지 않도록 app_stack cloud-init 구성을 제거했습니다. - 상세내용: k6 실행에 필요한 기본값은 secret이 아닌 Terraform 기본값과 output으로 관리하도록 정리했습니다. --- environment/load_test/main.tf | 62 ++++++++++++++++++++++++++++++ environment/load_test/output.tf | 25 ++++++++++++ environment/load_test/variables.tf | 36 +++++++++++++++++ environment/stage/main.tf | 2 - modules/app_stack/ec2.tf | 50 ------------------------ modules/app_stack/variables.tf | 12 ------ 6 files changed, 123 insertions(+), 64 deletions(-) diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index 2b5716f..35789d2 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -33,6 +33,21 @@ data "aws_subnets" "target" { } } +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + data "aws_db_instance" "prod" { db_instance_identifier = var.prod_rds_identifier } @@ -85,6 +100,53 @@ resource "aws_security_group_rule" "load_test_db_mysql" { source_security_group_id = each.value } +resource "aws_security_group" "load_generator" { + name = "sc-load-test-generator-sg" + description = "Security group for k6 load generator" + vpc_id = data.aws_subnet.stage_api.vpc_id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "solid-connection-load-test-generator-sg" + } +} + +resource "aws_instance" "load_generator" { + ami = data.aws_ami.ubuntu.id + instance_type = var.load_generator_instance_type + subnet_id = data.aws_instance.stage_api.subnet_id + vpc_security_group_ids = [aws_security_group.load_generator.id] + associate_public_ip_address = true + iam_instance_profile = var.load_generator_instance_profile_name + + root_block_device { + volume_size = var.load_generator_root_volume_size + volume_type = "gp3" + encrypted = true + } + + user_data = <<-EOF + #!/bin/bash + set -eux + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y curl jq + snap install amazon-ssm-agent --classic || true + systemctl enable snap.amazon-ssm-agent.amazon-ssm-agent.service || true + systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true + EOF + + tags = { + Name = "solid-connection-load-test-generator" + } +} + resource "aws_db_subnet_group" "load_test" { name = "sc-load-test-db-subnet-group" subnet_ids = data.aws_subnets.target.ids diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf index ee0bc25..901044a 100644 --- a/environment/load_test/output.tf +++ b/environment/load_test/output.tf @@ -67,3 +67,28 @@ output "prod_db_password_parameter_name" { description = "SSM SecureString parameter name containing the prod DB password" value = var.prod_db_password_parameter_name } + +output "load_generator_instance_id" { + description = "k6 load generator EC2 instance ID" + value = aws_instance.load_generator.id +} + +output "load_generator_private_ip" { + description = "k6 load generator private IP" + value = aws_instance.load_generator.private_ip +} + +output "load_generator_k6_dir" { + description = "Directory where k6 files are placed on the load generator" + value = var.load_generator_k6_dir +} + +output "load_test_target_base_url" { + description = "Default target base URL for k6" + value = var.load_test_target_base_url +} + +output "k6_prometheus_remote_write_url" { + description = "Default Prometheus remote-write URL for k6" + value = var.k6_prometheus_remote_write_url +} diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index efdfa28..9b2e2f6 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -86,3 +86,39 @@ variable "load_test_parameter_prefix" { type = string default = "/solid-connection/loadtest" } + +variable "load_generator_instance_type" { + description = "EC2 instance type for the k6 load generator" + type = string + default = "c7i.xlarge" +} + +variable "load_generator_instance_profile_name" { + description = "Existing IAM instance profile name for the k6 load generator. It must allow SSM RunCommand." + type = string + default = "solid-connection-load-test-generator" +} + +variable "load_generator_root_volume_size" { + description = "Root volume size in GiB for the k6 load generator" + type = number + default = 20 +} + +variable "load_generator_k6_dir" { + description = "Directory where k6 files are placed on the load generator" + type = string + default = "/home/ubuntu/solid-connection-load-test/k6" +} + +variable "load_test_target_base_url" { + description = "Default target base URL for k6" + type = string + default = "https://api.stage.solid-connection.com" +} + +variable "k6_prometheus_remote_write_url" { + description = "Default Prometheus remote-write URL for k6" + type = string + default = "http://132.145.83.182:9090/api/v1/write" +} diff --git a/environment/stage/main.tf b/environment/stage/main.tf index 8e60bb5..a0b0b44 100644 --- a/environment/stage/main.tf +++ b/environment/stage/main.tf @@ -41,6 +41,4 @@ module "stage_stack" { redis_version = var.redis_version redis_exporter_version = var.redis_exporter_version alloy_version = var.alloy_version - - enable_k6_files = true } diff --git a/modules/app_stack/ec2.tf b/modules/app_stack/ec2.tf index e1da678..6734a28 100644 --- a/modules/app_stack/ec2.tf +++ b/modules/app_stack/ec2.tf @@ -22,56 +22,6 @@ data "cloudinit_config" "app_init" { filename = "1_docker_install.sh" } - dynamic "part" { - for_each = var.enable_k6_files ? [1] : [] - - content { - content_type = "text/cloud-config" - content = yamlencode({ - bootcmd = [ - "mkdir -p ${var.k6_install_dir}/script" - ] - write_files = [ - { - path = "${var.k6_install_dir}/createPost.json" - owner = "ubuntu:ubuntu" - permissions = "0644" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/createPost.json") - }, - { - path = "${var.k6_install_dir}/updatePost.json" - owner = "ubuntu:ubuntu" - permissions = "0644" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/updatePost.json") - }, - { - path = "${var.k6_install_dir}/whole-user-flow.js" - owner = "ubuntu:ubuntu" - permissions = "0644" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/whole-user-flow.js") - }, - { - path = "${var.k6_install_dir}/set_up_xk6.sh" - owner = "ubuntu:ubuntu" - permissions = "0755" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/set_up_xk6.sh") - }, - { - path = "${var.k6_install_dir}/script/set-load-test.sh" - owner = "ubuntu:ubuntu" - permissions = "0755" - encoding = "b64" - content = filebase64("${path.module}/../../config/load-test/k6/script/set-load-test.sh") - } - ] - }) - filename = "2_k6_files.yml" - } - } } # API Server (EC2) diff --git a/modules/app_stack/variables.tf b/modules/app_stack/variables.tf index c1d199c..1c5028d 100644 --- a/modules/app_stack/variables.tf +++ b/modules/app_stack/variables.tf @@ -155,15 +155,3 @@ variable "alloy_version" { description = "Docker image tag for Grafana Alloy" type = string } - -variable "enable_k6_files" { - description = "Whether to place k6 load test files on the API server during cloud-init" - type = bool - default = false -} - -variable "k6_install_dir" { - description = "Directory where k6 load test files are placed" - type = string - default = "/home/ubuntu/solid-connection-load-test/k6" -} From c8a70cc6e5dfec36bc5150a26c9f47780c24cad2 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:05:12 +0900 Subject: [PATCH 09/11] =?UTF-8?q?feat:=20EC2=20=EA=B8=B0=EB=B0=98=20k6=20?= =?UTF-8?q?=EC=8B=A4=ED=96=89=20workflow=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: Load Test Run workflow를 추가해 k6 전용 EC2에서 부하를 생성하도록 했습니다. - 상세내용: loadtest workflow가 전용 AWS_LOAD_TEST_ROLE_ARN 변수를 사용하도록 분리했습니다. - 상세내용: start 스크립트에서 stage k6 동기화를 제거하고 생성된 부하 생성 EC2 정보를 출력하도록 수정했습니다. --- .github/workflows/load-test-run.yml | 84 ++++++++++ .github/workflows/load-test-start.yml | 13 +- .github/workflows/load-test-stop.yml | 2 +- scripts/load_test/run_k6.sh | 215 ++++++++++++++++++++++++++ scripts/load_test/start.sh | 56 +------ 5 files changed, 314 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/load-test-run.yml create mode 100644 scripts/load_test/run_k6.sh diff --git a/.github/workflows/load-test-run.yml b/.github/workflows/load-test-run.yml new file mode 100644 index 0000000..d8675e9 --- /dev/null +++ b/.github/workflows/load-test-run.yml @@ -0,0 +1,84 @@ +name: Load Test Run + +on: + workflow_dispatch: + inputs: + vus: + description: "k6 VUs" + required: true + default: "10" + type: string + iterations: + description: "k6 iterations per VU" + required: true + default: "10" + type: string + max_duration: + description: "k6 max duration" + required: true + default: "15m" + type: string + target_base_url: + description: "Target base URL. Empty uses Terraform default." + required: false + default: "" + type: string + prometheus_remote_write_url: + description: "Prometheus remote-write URL. Empty uses Terraform default." + required: false + default: "" + type: string + +permissions: + id-token: write + contents: read + +concurrency: + group: load-test-environment + cancel-in-progress: false + +env: + TF_VERSION: "1.10.5" + +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }} + aws-region: ap-northeast-2 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Run k6 on load generator + run: | + args=( + --vus "${{ inputs.vus }}" + --iterations "${{ inputs.iterations }}" + --max-duration "${{ inputs.max_duration }}" + ) + + if [ -n "${{ inputs.target_base_url }}" ]; then + args+=(--target-base-url "${{ inputs.target_base_url }}") + fi + + if [ -n "${{ inputs.prometheus_remote_write_url }}" ]; then + args+=(--prometheus-remote-write-url "${{ inputs.prometheus_remote_write_url }}") + fi + + bash scripts/load_test/run_k6.sh "${args[@]}" diff --git a/.github/workflows/load-test-start.yml b/.github/workflows/load-test-start.yml index 80125e4..8755782 100644 --- a/.github/workflows/load-test-start.yml +++ b/.github/workflows/load-test-start.yml @@ -13,6 +13,11 @@ on: required: true default: true type: boolean + load_generator_instance_type: + description: "k6 load generator EC2 instance type" + required: true + default: "c7i.xlarge" + type: string permissions: id-token: write @@ -37,7 +42,7 @@ jobs: - uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }} aws-region: ap-northeast-2 - uses: hashicorp/setup-terraform@v3 @@ -52,6 +57,12 @@ jobs: - name: Start load test environment run: | + export TF_VAR_load_generator_instance_type="${{ inputs.load_generator_instance_type }}" + + if [ -n "${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" ]; then + export TF_VAR_load_generator_instance_profile_name="${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" + fi + args=() if [ "${{ inputs.switch_stage_to_loadtest }}" = "true" ]; then diff --git a/.github/workflows/load-test-stop.yml b/.github/workflows/load-test-stop.yml index 9e5fba7..d5ceffa 100644 --- a/.github/workflows/load-test-stop.yml +++ b/.github/workflows/load-test-stop.yml @@ -37,7 +37,7 @@ jobs: - uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }} aws-region: ap-northeast-2 - uses: hashicorp/setup-terraform@v3 diff --git a/scripts/load_test/run_k6.sh b/scripts/load_test/run_k6.sh new file mode 100644 index 0000000..e2b0477 --- /dev/null +++ b/scripts/load_test/run_k6.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +set -euo pipefail + +TERRAFORM_DIR="environment/load_test" +LOCAL_K6_DIR="config/load-test/k6" +K6_SCRIPT="whole-user-flow.js" +TARGET_BASE_URL="" +PROMETHEUS_REMOTE_WRITE_URL="" +K6_VUS="10" +K6_ITERATIONS="10" +K6_MAX_DURATION="15m" +SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-3600}" + +usage() { + cat <<'EOF' +Usage: scripts/load_test/run_k6.sh [options] + +Options: + --terraform-dir PATH Default: environment/load_test + --local-k6-dir PATH Default: config/load-test/k6 + --script FILE Default: whole-user-flow.js + --target-base-url URL Default: Terraform output load_test_target_base_url + --prometheus-remote-write-url URL Default: Terraform output k6_prometheus_remote_write_url + --vus VALUE Default: 10 + --iterations VALUE Default: 10 + --max-duration VALUE Default: 15m + --ssm-command-timeout-seconds Default: 3600 + -h, --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --local-k6-dir) LOCAL_K6_DIR="$2"; shift 2 ;; + --script) K6_SCRIPT="$2"; shift 2 ;; + --target-base-url) TARGET_BASE_URL="$2"; shift 2 ;; + --prometheus-remote-write-url) PROMETHEUS_REMOTE_WRITE_URL="$2"; shift 2 ;; + --vus) K6_VUS="$2"; shift 2 ;; + --iterations) K6_ITERATIONS="$2"; shift 2 ;; + --max-duration) K6_MAX_DURATION="$2"; shift 2 ;; + --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Required command not found: $1" >&2 + exit 1 + fi +} + +require_command terraform +require_command aws +require_command jq +require_command base64 + +tf_output() { + terraform -chdir="$TERRAFORM_DIR" output -raw "$1" +} + +send_ssm_command() { + local instance_id="$1" + local comment="$2" + local commands_json="$3" + + local command_id + local started_at + command_id="$(aws ssm send-command \ + --instance-ids "$instance_id" \ + --document-name "AWS-RunShellScript" \ + --comment "$comment" \ + --parameters "$commands_json" \ + --query "Command.CommandId" \ + --output text)" + started_at="$(date +%s)" + + local status + while true; do + sleep 5 + status="$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "Status" \ + --output text 2>/dev/null || true)" + + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command timed out after ${SSM_COMMAND_TIMEOUT_SECONDS}s: $comment" >&2 + exit 1 + fi + + case "$status" in + Pending|InProgress|Delayed|"") continue ;; + Success) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "StandardOutputContent" \ + --output text || true + break + ;; + *) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command failed with status $status: $comment" >&2 + exit 1 + ;; + esac + done +} + +wait_for_ssm() { + local instance_id="$1" + local started_at + started_at="$(date +%s)" + + while true; do + local ping_status + ping_status="$(aws ssm describe-instance-information \ + --filters "Key=InstanceIds,Values=${instance_id}" \ + --query "InstanceInformationList[0].PingStatus" \ + --output text 2>/dev/null || true)" + + if [[ "$ping_status" == "Online" ]]; then + break + fi + + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + echo "SSM agent did not become online after ${SSM_COMMAND_TIMEOUT_SECONDS}s: ${instance_id}" >&2 + exit 1 + fi + + sleep 10 + done +} + +file_base64() { + base64 "$1" | tr -d '\n' +} + +sync_file() { + local instance_id="$1" + local target_dir="$2" + local relative_path="$3" + local source_path="${LOCAL_K6_DIR}/${relative_path}" + + if [[ ! -f "$source_path" ]]; then + echo "Missing k6 file: $source_path" >&2 + exit 1 + fi + + local commands_json + commands_json="$(jq -cn \ + --arg target "${target_dir}/${relative_path}" \ + --arg content "$(file_base64 "$source_path")" \ + '{ + commands: [ + "set -euo pipefail", + "mkdir -p \"$(dirname \"\($target)\")\"", + "printf %s \($content | @sh) | base64 -d > \($target | @sh)" + ] + }')" + + send_ssm_command "$instance_id" "Sync ${relative_path} to load generator" "$commands_json" +} + +terraform -chdir="$TERRAFORM_DIR" init + +load_generator_instance_id="$(tf_output load_generator_instance_id)" +load_generator_k6_dir="$(tf_output load_generator_k6_dir)" +tf_target_base_url="$(tf_output load_test_target_base_url)" +tf_prometheus_remote_write_url="$(tf_output k6_prometheus_remote_write_url)" + +TARGET_BASE_URL="${TARGET_BASE_URL:-$tf_target_base_url}" +PROMETHEUS_REMOTE_WRITE_URL="${PROMETHEUS_REMOTE_WRITE_URL:-$tf_prometheus_remote_write_url}" + +wait_for_ssm "$load_generator_instance_id" + +for relative_path in \ + "createPost.json" \ + "updatePost.json" \ + "whole-user-flow.js" \ + "set_up_xk6.sh" \ + "script/set-load-test.sh"; do + sync_file "$load_generator_instance_id" "$load_generator_k6_dir" "$relative_path" +done + +run_commands_json="$(jq -cn \ + --arg k6_dir "$load_generator_k6_dir" \ + --arg script "$K6_SCRIPT" \ + --arg target_base_url "$TARGET_BASE_URL" \ + --arg prometheus_url "$PROMETHEUS_REMOTE_WRITE_URL" \ + --arg vus "$K6_VUS" \ + --arg iterations "$K6_ITERATIONS" \ + --arg max_duration "$K6_MAX_DURATION" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($k6_dir)", + "chmod +x set_up_xk6.sh script/set-load-test.sh", + "chown -R ubuntu:ubuntu \($k6_dir)", + "if [ ! -x ./k6 ]; then sudo -u ubuntu -H ./set_up_xk6.sh; fi", + "sudo -u ubuntu -H env BASE_URL=\($target_base_url | @sh) K6_PROMETHEUS_RW_SERVER_URL=\($prometheus_url | @sh) K6_VUS=\($vus | @sh) K6_ITERATIONS=\($iterations | @sh) K6_MAX_DURATION=\($max_duration | @sh) ./k6 run \($script | @sh)" + ] + }')" + +send_ssm_command "$load_generator_instance_id" "Run k6 load test" "$run_commands_json" diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh index 21ab43d..e3a7414 100644 --- a/scripts/load_test/start.sh +++ b/scripts/load_test/start.sh @@ -12,8 +12,6 @@ LOADTEST_DB_PASSWORD_PARAMETER="" SWITCH_STAGE_TO_LOADTEST="false" STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" STAGE_COMPOSE_FILE="docker-compose.dev.yml" -STAGE_K6_DIR="/home/ubuntu/solid-connection-load-test/k6" -LOCAL_K6_DIR="config/load-test/k6" SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-1800}" SKIP_TERRAFORM_APPLY="false" SKIP_DATA_COPY="false" @@ -34,8 +32,6 @@ Options: --switch-stage-to-loadtest Restart stage app through SSM with dev,loadtest profiles --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev --stage-compose-file VALUE Default: docker-compose.dev.yml - --stage-k6-dir PATH Default: /home/ubuntu/solid-connection-load-test/k6 - --local-k6-dir PATH Default: config/load-test/k6 --ssm-command-timeout-seconds Default: 1800 --skip-terraform-apply --skip-data-copy @@ -56,8 +52,6 @@ while [[ $# -gt 0 ]]; do --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; - --stage-k6-dir) STAGE_K6_DIR="$2"; shift 2 ;; - --local-k6-dir) LOCAL_K6_DIR="$2"; shift 2 ;; --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; --skip-data-copy) SKIP_DATA_COPY="true"; shift ;; @@ -85,7 +79,6 @@ require_command() { require_command terraform require_command aws require_command jq -require_command base64 tf_output() { terraform -chdir="$TERRAFORM_DIR" output -raw "$1" @@ -140,51 +133,6 @@ send_ssm_command() { done } -file_base64() { - base64 "$1" | tr -d '\n' -} - -sync_stage_k6_files() { - local instance_id="$1" - local commands - commands="$(jq -cn \ - --arg target_dir "$STAGE_K6_DIR" \ - '{ commands: ["set -euo pipefail", "mkdir -p \($target_dir)/script"] }')" - - local relative_path - for relative_path in \ - "createPost.json" \ - "updatePost.json" \ - "whole-user-flow.js" \ - "set_up_xk6.sh" \ - "script/set-load-test.sh"; do - local source_path="${LOCAL_K6_DIR}/${relative_path}" - if [[ ! -f "$source_path" ]]; then - echo "Missing k6 file: $source_path" >&2 - exit 1 - fi - - commands="$(jq -cn \ - --argjson current "$commands" \ - --arg target "${STAGE_K6_DIR}/${relative_path}" \ - --arg content "$(file_base64 "$source_path")" \ - '$current | .commands += [ - "mkdir -p \"$(dirname \"\($target)\")\"", - "printf %s \($content | @sh) | base64 -d > \($target | @sh)" - ]')" - done - - commands="$(jq -cn \ - --argjson current "$commands" \ - --arg target_dir "$STAGE_K6_DIR" \ - '$current | .commands += [ - "chmod +x \($target_dir)/set_up_xk6.sh \($target_dir)/script/set-load-test.sh", - "chown -R ubuntu:ubuntu \($target_dir)" - ]')" - - send_ssm_command "$instance_id" "Sync k6 files to stage EC2" "$commands" -} - delete_temp_parameters() { aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" >/dev/null 2>&1 || true aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" >/dev/null 2>&1 || true @@ -204,6 +152,7 @@ prod_endpoint="$(tf_output prod_rds_endpoint)" prod_port="$(tf_output prod_rds_port)" loadtest_endpoint="$(tf_output load_test_rds_endpoint)" loadtest_port="$(tf_output load_test_rds_port)" +load_generator_instance_id="$(tf_output load_generator_instance_id)" loadtest_db_name="$(tf_output load_test_db_name)" tf_prod_db_username_parameter="$(tf_output prod_db_username_parameter_name)" tf_prod_db_password_parameter="$(tf_output prod_db_password_parameter_name)" @@ -294,8 +243,6 @@ if [[ "$SKIP_DATA_COPY" != "true" ]]; then fi if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then - sync_stage_k6_files "$stage_instance_id" - stage_commands_json="$(jq -cn \ --arg app_dir "$STAGE_APP_DIR" \ --arg compose_file "$STAGE_COMPOSE_FILE" \ @@ -318,5 +265,6 @@ fi echo "Load test environment is ready." echo "RDS endpoint: ${loadtest_endpoint}:${loadtest_port}" +echo "Load generator instance: ${load_generator_instance_id}" echo "Stage instance: ${stage_instance_id}" echo "Stage public IP: ${stage_public_ip}" From b097ca95d25e339fe4dc73bc5779f3359a96daff Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:05:22 +0900 Subject: [PATCH 10/11] =?UTF-8?q?fix:=20k6=20=EC=8B=A4=ED=96=89=20?= =?UTF-8?q?=EC=84=A4=EC=A0=95=EA=B3=BC=20=EC=9D=91=EB=8B=B5=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: Prometheus remote-write 설정을 환경 변수 기반으로 일관되게 export하도록 수정했습니다. - 상세내용: k6 VU, iteration, duration, target URL을 실행 시 주입할 수 있도록 변경했습니다. - 상세내용: 대학, 어학 점수, GPA 응답이 비어 있을 때 명확히 fail하도록 검증을 추가했습니다. --- config/load-test/k6/set_up_xk6.sh | 4 ++-- config/load-test/k6/whole-user-flow.js | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/load-test/k6/set_up_xk6.sh b/config/load-test/k6/set_up_xk6.sh index 01ad6d6..eb1f254 100644 --- a/config/load-test/k6/set_up_xk6.sh +++ b/config/load-test/k6/set_up_xk6.sh @@ -11,8 +11,8 @@ export GOPATH=${BASE_DIR}/go-workspace export PATH=$PATH:$GOROOT/bin:$GOPATH/bin export XK6_BIN=${GOPATH}/bin/xk6 export K6_OUT=xk6-prometheus-rw -export K6_PROMETHEUS_RW_SERVER_URL=http://132.145.83.182:9090/api/v1/write -export K6_PROMETHEUS_RW_TREND_STATS="p(90),p(95),p(99),avg,min,max" +export K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL:-http://132.145.83.182:9090/api/v1/write} +export K6_PROMETHEUS_RW_TREND_STATS="${K6_PROMETHEUS_RW_TREND_STATS:-p(90),p(95),p(99),avg,min,max}" { echo "export BASE_DIR=${BASE_DIR}" diff --git a/config/load-test/k6/whole-user-flow.js b/config/load-test/k6/whole-user-flow.js index a40f82c..8195806 100644 --- a/config/load-test/k6/whole-user-flow.js +++ b/config/load-test/k6/whole-user-flow.js @@ -13,16 +13,16 @@ const time = (() => { return `${mm}/${dd} ${hh}:${min}`; })(); -const BASE_URL = 'https://api.stage.solid-connection.com'; +const BASE_URL = __ENV.BASE_URL || 'https://api.stage.solid-connection.com'; const testId = 'whole-user-flow'; export const options = { scenarios: { user_flow: { executor: 'per-vu-iterations', // VU별 반복 - vus: 10, // VU - iterations: 10, // VU 한 명당 실행할 횟수 - maxDuration: '15m', // 여유로 잡아 두기 + vus: Number(__ENV.K6_VUS || 10), + iterations: Number(__ENV.K6_ITERATIONS || 10), + maxDuration: __ENV.K6_MAX_DURATION || '15m', }, }, tags: { From 4ad54c46193cd550e2578a90067172e491e12c22 Mon Sep 17 00:00:00 2001 From: Yeonri Date: Wed, 6 May 2026 17:05:28 +0900 Subject: [PATCH 11/11] =?UTF-8?q?docs:=20EC2=20=EA=B8=B0=EB=B0=98=20?= =?UTF-8?q?=EB=B6=80=ED=95=98=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=A0=88?= =?UTF-8?q?=EC=B0=A8=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 상세내용: Start, Run, Stop workflow 기준의 부하 테스트 실행 흐름을 문서화했습니다. - 상세내용: secret에 새로 추가할 값이 없고 민감하지 않은 값은 workflow 입력과 기본값으로 관리한다는 점을 명시했습니다. - 상세내용: stage EC2가 아닌 k6 전용 EC2에서 부하를 생성하는 구조를 설명했습니다. --- scripts/load_test/README.md | 61 ++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md index 3647c44..dacc960 100644 --- a/scripts/load_test/README.md +++ b/scripts/load_test/README.md @@ -1,13 +1,32 @@ # 부하 테스트 자동화 -부하 테스트용 임시 RDS를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작과 종료는 GitHub Actions에서 수동으로 실행합니다. +부하 테스트용 임시 RDS와 k6 전용 EC2를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작, 실행, 종료는 GitHub Actions에서 수동으로 실행합니다. + +## secret에 필요한 값 + +EC2 기반 k6 실행을 위해 새로 `config/secrets/load_test.tfvars`에 추가해야 하는 값은 없습니다. + +기존 loadtest Terraform에 필요한 민감값만 secret에 둡니다. + +- `load_test_db_username_parameter_name` +- `load_test_db_password_parameter_name` + +아래 값들은 민감값이 아니므로 secret에 넣지 않고 Terraform 기본값이나 GitHub Actions 입력값으로 처리합니다. + +- k6 전용 EC2 instance type: 기본값 `c7i.xlarge` +- k6 전용 EC2 IAM instance profile 이름: 기본값 `solid-connection-load-test-generator` +- k6 target URL: 기본값 `https://api.stage.solid-connection.com` +- k6 VU, iterations, max duration: **Load Test Run** workflow 입력값 +- Prometheus remote-write URL: 기본값 사용 또는 **Load Test Run** workflow 입력값 + +기본 instance profile 이름을 쓰지 않을 경우 secret이 아니라 GitHub Actions variable 또는 `TF_VAR_load_generator_instance_profile_name`으로 덮어씁니다. ## 원칙 - 사람이 로컬에서 `terraform apply` 또는 `terraform destroy`를 직접 실행하지 않습니다. -- 시작은 **Actions > Load Test Start** workflow로 실행합니다. -- 종료는 **Actions > Load Test Stop** workflow로 실행합니다. -- SSH private key는 사용하지 않습니다. stage/prod EC2 작업은 SSM RunCommand로 실행합니다. +- stage EC2는 부하를 받는 대상이므로 stage EC2에서 k6를 실행하지 않습니다. +- k6는 loadtest Terraform이 생성한 별도 EC2에서 실행합니다. +- SSH private key는 사용하지 않습니다. stage/prod/load-generator EC2 작업은 SSM RunCommand로 실행합니다. ## 시작 @@ -25,13 +44,27 @@ workflow는 GitHub Actions runner에서 `environment/load_test`의 Terraform을 ## 시작 시 수행 작업 1. GitHub Actions가 `environment/load_test`에서 Terraform apply를 실행합니다. -2. loadtest RDS와 보안 그룹을 생성합니다. RDS는 stage EC2가 속한 VPC/subnet 기준으로 생성됩니다. +2. loadtest RDS, 보안 그룹, k6 전용 EC2를 생성합니다. 3. loadtest datasource 값을 Parameter Store에 작성합니다. 4. prod DB와 loadtest DB 접속 정보를 Parameter Store에서 읽습니다. 5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS로 복원합니다. -6. stage EC2에 k6 파일을 `/home/ubuntu/solid-connection-load-test/k6` 경로로 동기화합니다. -7. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. -8. 데이터 이관용 임시 Parameter Store 값을 삭제합니다. +6. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +7. 데이터 이관용 임시 Parameter Store 값을 삭제합니다. + +## 실행 + +1. GitHub에서 **Actions > Load Test Run**을 엽니다. +2. **Run workflow**를 클릭합니다. +3. VU, iterations, max duration을 입력하고 실행합니다. + +workflow는 `scripts/load_test/run_k6.sh`를 실행합니다. + +실행 시 수행 작업: + +1. Terraform output에서 k6 전용 EC2 ID와 기본 실행값을 읽습니다. +2. SSM RunCommand로 k6 파일을 k6 전용 EC2에 배치합니다. +3. k6 전용 EC2에서 `set_up_xk6.sh`로 k6 binary를 준비합니다. +4. k6 전용 EC2에서 `whole-user-flow.js`를 실행합니다. ## 종료 @@ -44,12 +77,10 @@ workflow는 GitHub Actions runner에서 `environment/load_test`의 Terraform을 - `restore_stage_dev`: stage 앱을 기존 dev compose 구성으로 되돌립니다. - `destroy_rds`: loadtest Terraform stack을 제거합니다. -workflow는 `scripts/load_test/stop.sh`를 실행하고, 선택값에 따라 stage 복구와 Terraform destroy를 수행합니다. +workflow는 `scripts/load_test/stop.sh`를 실행하고, 선택값에 따라 stage 복구와 Terraform destroy를 수행합니다. Terraform destroy에는 loadtest RDS와 k6 전용 EC2 제거가 포함됩니다. ## k6 파일 -stage EC2를 새로 생성하는 경우 Terraform cloud-init이 k6 파일을 배치합니다. 기존 stage EC2는 cloud-init이 다시 실행되지 않으므로, **Load Test Start** workflow가 실행될 때 SSM으로 k6 파일을 다시 동기화합니다. - 포함 파일: - `createPost.json` @@ -58,13 +89,7 @@ stage EC2를 새로 생성하는 경우 Terraform cloud-init이 k6 파일을 배 - `set_up_xk6.sh` - `script/set-load-test.sh` -stage EC2에 접속해 수동으로 실행해야 한다면 다음 경로에서 실행합니다. - -```bash -cd /home/ubuntu/solid-connection-load-test/k6 -./set_up_xk6.sh -./script/set-load-test.sh -``` +이 파일들은 stage EC2가 아니라 k6 전용 EC2에 동기화됩니다. ## 참고