diff --git a/.github/workflows/load-test-run.yml b/.github/workflows/load-test-run.yml new file mode 100644 index 0000000..d8675e9 --- /dev/null +++ b/.github/workflows/load-test-run.yml @@ -0,0 +1,84 @@ +name: Load Test Run + +on: + workflow_dispatch: + inputs: + vus: + description: "k6 VUs" + required: true + default: "10" + type: string + iterations: + description: "k6 iterations per VU" + required: true + default: "10" + type: string + max_duration: + description: "k6 max duration" + required: true + default: "15m" + type: string + target_base_url: + description: "Target base URL. Empty uses Terraform default." + required: false + default: "" + type: string + prometheus_remote_write_url: + description: "Prometheus remote-write URL. Empty uses Terraform default." + required: false + default: "" + type: string + +permissions: + id-token: write + contents: read + +concurrency: + group: load-test-environment + cancel-in-progress: false + +env: + TF_VERSION: "1.10.5" + +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }} + aws-region: ap-northeast-2 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Run k6 on load generator + run: | + args=( + --vus "${{ inputs.vus }}" + --iterations "${{ inputs.iterations }}" + --max-duration "${{ inputs.max_duration }}" + ) + + if [ -n "${{ inputs.target_base_url }}" ]; then + args+=(--target-base-url "${{ inputs.target_base_url }}") + fi + + if [ -n "${{ inputs.prometheus_remote_write_url }}" ]; then + args+=(--prometheus-remote-write-url "${{ inputs.prometheus_remote_write_url }}") + fi + + bash scripts/load_test/run_k6.sh "${args[@]}" diff --git a/.github/workflows/load-test-start.yml b/.github/workflows/load-test-start.yml new file mode 100644 index 0000000..8755782 --- /dev/null +++ b/.github/workflows/load-test-start.yml @@ -0,0 +1,76 @@ +name: Load Test Start + +on: + workflow_dispatch: + inputs: + switch_stage_to_loadtest: + description: "Restart stage app with dev,loadtest profiles" + required: true + default: true + type: boolean + copy_prod_data: + description: "Copy prod RDS data to load test RDS" + required: true + default: true + type: boolean + load_generator_instance_type: + description: "k6 load generator EC2 instance type" + required: true + default: "c7i.xlarge" + type: string + +permissions: + id-token: write + contents: read + +concurrency: + group: load-test-environment + cancel-in-progress: false + +env: + TF_VERSION: "1.10.5" + +jobs: + start: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }} + aws-region: ap-northeast-2 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Start load test environment + run: | + export TF_VAR_load_generator_instance_type="${{ inputs.load_generator_instance_type }}" + + if [ -n "${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" ]; then + export TF_VAR_load_generator_instance_profile_name="${{ vars.LOAD_GENERATOR_INSTANCE_PROFILE_NAME }}" + fi + + args=() + + if [ "${{ inputs.switch_stage_to_loadtest }}" = "true" ]; then + args+=(--switch-stage-to-loadtest) + fi + + if [ "${{ inputs.copy_prod_data }}" != "true" ]; then + args+=(--skip-data-copy) + fi + + bash scripts/load_test/start.sh "${args[@]}" diff --git a/.github/workflows/load-test-stop.yml b/.github/workflows/load-test-stop.yml new file mode 100644 index 0000000..d5ceffa --- /dev/null +++ b/.github/workflows/load-test-stop.yml @@ -0,0 +1,65 @@ +name: Load Test Stop + +on: + workflow_dispatch: + inputs: + restore_stage_dev: + description: "Restart stage app with dev profile" + required: true + default: true + type: boolean + destroy_rds: + description: "Destroy load test Terraform stack" + required: true + default: true + type: boolean + +permissions: + id-token: write + contents: read + +concurrency: + group: load-test-environment + cancel-in-progress: false + +env: + TF_VERSION: "1.10.5" + +jobs: + stop: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + token: ${{ secrets.GH_PAT }} + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }} + aws-region: ap-northeast-2 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + terraform_wrapper: false + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Stop load test environment + run: | + args=() + + if [ "${{ inputs.restore_stage_dev }}" = "true" ]; then + args+=(--restore-stage-dev) + fi + + if [ "${{ inputs.destroy_rds }}" != "true" ]; then + args+=(--skip-terraform-destroy) + fi + + bash scripts/load_test/stop.sh "${args[@]}" diff --git a/config/load-test/k6/createPost.json b/config/load-test/k6/createPost.json new file mode 100644 index 0000000..e08b0d2 --- /dev/null +++ b/config/load-test/k6/createPost.json @@ -0,0 +1,7 @@ +{ + "boardCode": "FREE", + "postCategory": "자유", + "title": "수강신청 어떻게 하나요?", + "content": "수강신청 방법을 잘 모르겠어요.", + "isQuestion": false +} diff --git a/config/load-test/k6/script/set-load-test.sh b/config/load-test/k6/script/set-load-test.sh new file mode 100644 index 0000000..b401b2d --- /dev/null +++ b/config/load-test/k6/script/set-load-test.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +# 작업 디렉터리 설정 +WORKDIR="/home/ubuntu" + +####################################################################### +# set-load-test.sh +# 사용 예: SQL_FILE_BASENAME=regular.sql ./set-load-test.sh +####################################################################### + +# 0. 필수 값 검증 +SQL_BASENAME="${SQL_FILE_BASENAME:-regular.sql}" +if [[ -z "$SQL_BASENAME" ]]; then + echo "❌ 사용할 덤프 파일이 입력되지 않았습니다." >&2 + echo "SQL_FILE_BASENAME={dump 파일 이름} ./set-load-test.sh 형식으로 인자를 전달해야합니다." >&2 + exit 1 +fi + +SQL_SRC="$WORKDIR/load-test-setting/db/${SQL_BASENAME}" +if [[ ! -f "$SQL_SRC" ]]; then + echo "❌ 덤프 파일이 파일 시스템에 존재하지 않습니다: $SQL_SRC" >&2 + exit 2 +fi + +# 1. 기존 어플리케이션, DB 중지 +docker compose -f "$WORKDIR/solid-connection-dev/docker-compose.dev.yml" down \ + || { echo "❌ 어플리케이션 도커 중지 실패: $WORKDIR/solid-connection-dev/docker-compose.dev.yml" >&2; exit 3; } +docker compose -f "$WORKDIR/mysql/docker-compose.mysql.yml" down \ + || { echo "❌ MySQL 도커 중지 실패: $WORKDIR/mysql/docker-compose.mysql.yml" >&2; exit 4; } + +# 2. 부하 테스트용 DB 실행 +docker compose -f "$WORKDIR/load-test-setting/docker-compose.load-test.yml" up -d \ + || { echo "❌ 부하 테스트용 DB 실행 실패: docker-compose.load-test.yml" >&2; exit 5; } + +# 3. MySQL 준비 대기 (최대 30초) +CONTAINER_NAME="load-test-db" +echo "⏳ MySQL이 준비될 때까지 대기 중 (최대 30초)..." +start_time=$(date +%s) +while ! docker exec "$CONTAINER_NAME" sh -c 'mysqladmin ping -h "127.0.0.1" --silent'; do + elapsed=$(( $(date +%s) - start_time )) + if [[ $elapsed -ge 30 ]]; then + echo "❌ MySQL 준비 시간 초과 (30초)" >&2 + exit 1 + fi + printf "." + sleep 1 +done +echo "✔️ MySQL 준비 완료." + +# 4. dump 주입 +echo "📥 dump 파일 복사 → 컨테이너: $CONTAINER_NAME" +docker cp "$SQL_SRC" "${CONTAINER_NAME}:/tmp/dump.sql" \ + || { echo "❌ dump 복사 실패: $SQL_SRC → $CONTAINER_NAME:/tmp/dump.sql" >&2; exit 6; } + +echo "⚙️ dump 이식" +docker exec -i "$CONTAINER_NAME" sh -c 'mysql -u root -proot < /tmp/dump.sql' \ + || { echo "❌ dump 이식 실패: 컨테이너 $CONTAINER_NAME" >&2; exit 7; } + +# 5. 어플리케이션 다시 실행 +docker compose -f "$WORKDIR/solid-connection-dev/docker-compose.dev.yml" up -d \ + || { echo "❌ 어플리케이션 재시작 실패: $WORKDIR/solid-connection-dev/docker-compose.dev.yml" >&2; exit 8; } + +echo "✅ 부하 테스트용 DB에 연결된 어플리케이션 실행 완료!" diff --git a/config/load-test/k6/set_up_xk6.sh b/config/load-test/k6/set_up_xk6.sh new file mode 100644 index 0000000..eb1f254 --- /dev/null +++ b/config/load-test/k6/set_up_xk6.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -euo pipefail + +trap 'echo "xk6 setup failed" >&2' ERR + +export GO_VERSION=1.22.2 +export BASE_DIR=/home/ubuntu/solid-connection-load-test/k6 +export GOROOT=${BASE_DIR}/go +export GOPATH=${BASE_DIR}/go-workspace +export PATH=$PATH:$GOROOT/bin:$GOPATH/bin +export XK6_BIN=${GOPATH}/bin/xk6 +export K6_OUT=xk6-prometheus-rw +export K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL:-http://132.145.83.182:9090/api/v1/write} +export K6_PROMETHEUS_RW_TREND_STATS="${K6_PROMETHEUS_RW_TREND_STATS:-p(90),p(95),p(99),avg,min,max}" + +{ + echo "export BASE_DIR=${BASE_DIR}" + echo "export GOROOT=${GOROOT}" + echo "export GOPATH=${GOPATH}" + echo "export PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin" + echo "export XK6_BIN=${GOPATH}/bin/xk6" + echo "export K6_OUT=xk6-prometheus-rw" + echo "export K6_PROMETHEUS_RW_SERVER_URL=${K6_PROMETHEUS_RW_SERVER_URL}" + echo "export K6_PROMETHEUS_RW_TREND_STATS=\"${K6_PROMETHEUS_RW_TREND_STATS}\"" +} >> ~/.bashrc + +echo "Create and enter ${BASE_DIR}" +mkdir -p "$BASE_DIR" +cd "$BASE_DIR" + +echo "Download Go ${GO_VERSION}" +curl -OL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" + +echo "Extract Go" +tar -xzf "go${GO_VERSION}.linux-amd64.tar.gz" +rm "go${GO_VERSION}.linux-amd64.tar.gz" + +echo "Go version: $(go version)" + +echo "Install xk6" +go install go.k6.io/xk6/cmd/xk6@latest + +echo "xk6 installed: ${XK6_BIN}" +"$XK6_BIN" --help > /dev/null && echo "xk6 executable is available" + +echo "Build k6 with Prometheus remote-write output" +"$XK6_BIN" build --with github.com/grafana/xk6-output-prometheus-remote@latest + +echo "Build complete: $(pwd)/k6" +ls -lh ./k6 + +echo "xk6 setup completed" diff --git a/config/load-test/k6/updatePost.json b/config/load-test/k6/updatePost.json new file mode 100644 index 0000000..660f0a8 --- /dev/null +++ b/config/load-test/k6/updatePost.json @@ -0,0 +1,5 @@ +{ + "postCategory": "자유", + "title": "수강신청 어떻게 하나요?", + "content": "수강신청 방법을 잘 알겠어요." +} diff --git a/config/load-test/k6/whole-user-flow.js b/config/load-test/k6/whole-user-flow.js new file mode 100644 index 0000000..8195806 --- /dev/null +++ b/config/load-test/k6/whole-user-flow.js @@ -0,0 +1,386 @@ +import http from 'k6/http'; +import { sleep, check, fail } from 'k6'; + +// KST +const now = new Date(); +const kstOffset = 9 * 60; // 분 단위 +const d = new Date(now.getTime() + kstOffset * 60 * 1000); +const kst = d.toISOString().slice(0, 16); // "yyyy-mm-ddTHH:MM" + +// "mm/dd HH:MM" +const time = (() => { + const [yyyy, mm, dd, hh, min] = kst.split(/[-T:]/); + return `${mm}/${dd} ${hh}:${min}`; +})(); + +const BASE_URL = __ENV.BASE_URL || 'https://api.stage.solid-connection.com'; +const testId = 'whole-user-flow'; + +export const options = { + scenarios: { + user_flow: { + executor: 'per-vu-iterations', // VU별 반복 + vus: Number(__ENV.K6_VUS || 10), + iterations: Number(__ENV.K6_ITERATIONS || 10), + maxDuration: __ENV.K6_MAX_DURATION || '15m', + }, + }, + tags: { + testid: testId, + time: time, + }, +}; + +function authHeadersWithTags(token) { + return { + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json; charset=utf-8', + }, + tags: { + ...options.tags, + time: time, + }, + }; +} + +function login() { + // __VU: 현재 VU 인덱스 + const email = `user${__VU}@example.com`; + const password = 'password'; + + const res = http.post(`${BASE_URL}/auth/email/sign-in`, JSON.stringify({ + email: email, + password: password, + }), { + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + tags: { + name: '/auth/email/sign-in', + } + }); + if (res.status !== 200) { + fail('로그인 실패'); + } + return res.json('accessToken'); +} + +// universites +function getRecommendedUniversities(auth) { + http.get(`${BASE_URL}/universities/recommend`, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/recommend', + }, +}); +} +function likeUniversity(id, auth) { + http.post(`${BASE_URL}/universities/${id}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/{id}/like', + }, + }); +} +function isLikedUniversity(id, auth) { + http.get(`${BASE_URL}/universities/${id}/like`, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/{id}/like', + }, + }); +} +function getLikedUniversities(auth) { + http.get(`${BASE_URL}/universities/like`, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/like', + }, +}); +} +function cancelLikeUniversity(id, auth) { + http.del(`${BASE_URL}/universities/${id}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/universities/{id}/like', + }, + }); +} +function searchUniversities(params) { + return http.get(`${BASE_URL}/universities/search?${params}`, { + tags: { + name: '/universities/search?{params}', + }, +}); +} +function getDetailedUniversityInfo(id) { + http.get(`${BASE_URL}/universities/${id}`, { + tags: { + name: '/universities/{id}', + }, + }); +} + +// my +function getMyInfo(auth) { + http.get(`${BASE_URL}/my`, { + ...auth, + tags: { + ...auth.tags, + name: '/my', + }, +}); +} + +// users +function checkNicknameExists(nickname) { + http.get(`${BASE_URL}/users/exists?nickname=${nickname}`, { + tags: { + name: '/users/exists?nickname={nickname}', + }, + }); +} + +// boards +function getBoards(auth) { + http.get(`${BASE_URL}/boards`, { + ...auth, + tags: { + ...auth.tags, + name: '/boards', + }, +}); +} +function getPostsByBoard(boardCode, auth) { + http.get(`${BASE_URL}/boards/${boardCode}`, { + ...auth, + tags: { + ...auth.tags, + name: '/boards/{boardCode}', + }, + }); +} + +// posts +const createPostJson = open('./createPost.json', 'b'); +function createPost(token) { + const formData = { + postCreateRequest: http.file(createPostJson, 'post.json', 'application/json'), + }; + const res = http.post(`${BASE_URL}/posts`, formData, { + headers: { + Authorization: `Bearer ${token}` + }, + tags: { + testid: testId, + time: time, + name: '/posts' + }, + }); + return res.json('id'); +} +const updatePostJson = open('./updatePost.json', 'b'); +function updatePost(postId, token) { + const formData = { + postUpdateRequest: http.file(updatePostJson, 'post.json', 'application/json'), + }; + http.patch(`${BASE_URL}/posts/${postId}`, formData, { + headers: { + Authorization: `Bearer ${token}` + }, + tags: { + testid: testId, + time: time, + name: '/posts/{postId}' + }, + }); +} +function getPostDetail(postId, auth) { + http.get(`${BASE_URL}/posts/${postId}`, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}', + }, + }); +} +function likePost(postId, auth) { + http.post(`${BASE_URL}/posts/${postId}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}/like', + }, + }); +} +function cancelLikePost(postId, auth) { + http.del(`${BASE_URL}/posts/${postId}/like`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}/like', + }, + }); +} +function deletePost(postId, auth) { + http.del(`${BASE_URL}/posts/${postId}`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/posts/{postId}', + }, + }); +} + +// comments +function createComment(postId, auth) { + const res = http.post( + `${BASE_URL}/comments`, + JSON.stringify({ postId, content: '댓글', parentId: null }), + { + ...auth, + tags: { + ...auth.tags, + name: '/comments', + }, + }); + return res.json('id'); +} +function updateComment(commentId, auth) { + http.patch( + `${BASE_URL}/comments/${commentId}`, + JSON.stringify({ content: '댓글 수정' }), + { + ...auth, + tags: { + ...auth.tags, + name: '/comments/{commentId}', + }, + } + ); +} +function deleteComment(commentId, auth) { + http.del(`${BASE_URL}/comments/${commentId}`, null, { + ...auth, + tags: { + ...auth.tags, + name: '/comments/{commentId}', + }, + }); +} + +// scores +function getLanguageTests(auth) { + return http.get(`${BASE_URL}/scores/language-tests`, { + ...auth, + tags: { + ...auth.tags, + name: '/scores/language-tests', + }, + }); +} +function getGPAs(auth) { + return http.get(`${BASE_URL}/scores/gpas`, { + ...auth, + tags: { + ...auth.tags, + name: '/scores/gpas', + }, + }); +} + +function requireArray(value, name) { + if (!Array.isArray(value) || value.length === 0) { + fail(`${name} response is empty or invalid`); + } + return value; +} + +function requireId(value, name) { + if (!value || value.id === undefined || value.id === null) { + fail(`${name} response does not contain id`); + } + return value.id; +} + +// applications +function apply(gpaScoreId, languageTestScoreId, universityId, auth) { + http.post(`${BASE_URL}/applications`, JSON.stringify({ + gpaScoreId: gpaScoreId, + languageTestScoreId: languageTestScoreId, + universityChoiceRequest: { + firstChoiceUniversityId: universityId, + secondChoiceUniversityId: null, + thirdChoiceUniversityId: null + }, + }), { + ...auth, + tags: { + ...auth.tags, + name: '/applications', + }, + }); +} + +function getCompetitors(auth) { + http.get(`${BASE_URL}/applications/competitors`, { + ...auth, + tags: { + ...auth.tags, + name: '/applications/competitors', + }, + }); +} + +export default function () { + checkNicknameExists(encodeURIComponent('loadtest-user')); + const token = login(); + const auth = authHeadersWithTags(token); + + + getRecommendedUniversities(auth); + + const uniSearchRes = searchUniversities(''); // 이번학기 열린 대학 중 랜덤하게 id 가져오기 + const uniList = requireArray(uniSearchRes.json(), 'universities/search'); + const universityId = requireId(uniList[Math.floor(Math.random() * uniList.length)], 'universities/search item'); + + likeUniversity(universityId, auth); + isLikedUniversity(universityId, auth); + getLikedUniversities(auth); + cancelLikeUniversity(universityId, auth); + getDetailedUniversityInfo(universityId); + + getMyInfo(auth); + + getBoards(auth); + getPostsByBoard('FREE', auth); + + const postId = createPost(token); + updatePost(postId, token); + getPostDetail(postId, auth); + likePost(postId, auth); + cancelLikePost(postId, auth); + + const commentId = createComment(postId, auth); + updateComment(commentId, auth); + deleteComment(commentId, auth); + + deletePost(postId, auth); + + const langRes = getLanguageTests(auth); + const langList = requireArray(langRes.json().languageTestScoreStatusResponseList, 'scores/language-tests'); + const languageTestScoreId = requireId(langList[0], 'scores/language-tests item'); + + const gpaRes = getGPAs(auth); + const gpaList = requireArray(gpaRes.json().gpaScoreStatusResponseList, 'scores/gpas'); + const gpaScoreId = requireId(gpaList[0], 'scores/gpas item'); + + apply(gpaScoreId, languageTestScoreId, universityId, auth); + getCompetitors(auth); + + sleep(1); +} diff --git a/config/secrets b/config/secrets index f88a84c..b11ecad 160000 --- a/config/secrets +++ b/config/secrets @@ -1 +1 @@ -Subproject commit f88a84cdab72136d294614fd1e2c855c4a026c43 +Subproject commit b11ecadb9329eb4e84b93e98e94d8ab685605caa diff --git a/environment/load_test/main.tf b/environment/load_test/main.tf index 995074f..35789d2 100644 --- a/environment/load_test/main.tf +++ b/environment/load_test/main.tf @@ -1 +1,206 @@ -# TODO:: 부하 테스트용 EC2 인스턴스 및 보안 그룹 리소스 정의 필요 +data "aws_instance" "prod_api" { + filter { + name = "tag:Name" + values = [var.prod_api_instance_name] + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + +data "aws_instance" "stage_api" { + filter { + name = "tag:Name" + values = [var.stage_api_instance_name] + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + +data "aws_subnet" "stage_api" { + id = data.aws_instance.stage_api.subnet_id +} + +data "aws_subnets" "target" { + filter { + name = "vpc-id" + values = [data.aws_subnet.stage_api.vpc_id] + } +} + +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +data "aws_db_instance" "prod" { + db_instance_identifier = var.prod_rds_identifier +} + +data "aws_ssm_parameter" "db_root_username" { + name = var.load_test_db_username_parameter_name +} + +data "aws_ssm_parameter" "db_root_password" { + name = var.load_test_db_password_parameter_name + with_decryption = true +} + +locals { + db_root_username = data.aws_ssm_parameter.db_root_username.value + db_root_password = data.aws_ssm_parameter.db_root_password.value + + source_security_group_ids = setunion( + data.aws_instance.prod_api.vpc_security_group_ids, + data.aws_instance.stage_api.vpc_security_group_ids + ) +} + +resource "aws_security_group" "load_test_db" { + name = "sc-load-test-db-sg" + description = "Security group for load test RDS" + vpc_id = data.aws_subnet.stage_api.vpc_id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "solid-connection-load-test-db-sg" + } +} + +resource "aws_security_group_rule" "load_test_db_mysql" { + for_each = local.source_security_group_ids + + type = "ingress" + description = "MySQL from prod/stage API server" + from_port = 3306 + to_port = 3306 + protocol = "tcp" + security_group_id = aws_security_group.load_test_db.id + source_security_group_id = each.value +} + +resource "aws_security_group" "load_generator" { + name = "sc-load-test-generator-sg" + description = "Security group for k6 load generator" + vpc_id = data.aws_subnet.stage_api.vpc_id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "solid-connection-load-test-generator-sg" + } +} + +resource "aws_instance" "load_generator" { + ami = data.aws_ami.ubuntu.id + instance_type = var.load_generator_instance_type + subnet_id = data.aws_instance.stage_api.subnet_id + vpc_security_group_ids = [aws_security_group.load_generator.id] + associate_public_ip_address = true + iam_instance_profile = var.load_generator_instance_profile_name + + root_block_device { + volume_size = var.load_generator_root_volume_size + volume_type = "gp3" + encrypted = true + } + + user_data = <<-EOF + #!/bin/bash + set -eux + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y curl jq + snap install amazon-ssm-agent --classic || true + systemctl enable snap.amazon-ssm-agent.amazon-ssm-agent.service || true + systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true + EOF + + tags = { + Name = "solid-connection-load-test-generator" + } +} + +resource "aws_db_subnet_group" "load_test" { + name = "sc-load-test-db-subnet-group" + subnet_ids = data.aws_subnets.target.ids + + tags = { + Name = "solid-connection-load-test-db-subnet-group" + } +} + +resource "aws_db_instance" "load_test" { + identifier = var.rds_identifier + allocated_storage = var.allocated_storage + engine = "mysql" + engine_version = var.db_engine_version + instance_class = var.db_instance_class + db_name = var.db_name + username = local.db_root_username + password = local.db_root_password + parameter_group_name = var.db_parameter_group_name + db_subnet_group_name = aws_db_subnet_group.load_test.name + vpc_security_group_ids = [aws_security_group.load_test_db.id] + publicly_accessible = false + skip_final_snapshot = true + copy_tags_to_snapshot = true + deletion_protection = false + backup_retention_period = 0 + apply_immediately = true + storage_encrypted = true + kms_key_id = var.kms_key_arn + + tags = { + Name = var.rds_identifier + } +} + +resource "aws_ssm_parameter" "load_test_datasource_url" { + name = "${var.load_test_parameter_prefix}/spring.datasource.url" + type = "String" + value = "jdbc:mysql://${aws_db_instance.load_test.address}:${aws_db_instance.load_test.port}/${var.db_name}?serverTimezone=Asia/Seoul&characterEncoding=UTF-8" + overwrite = true +} + +resource "aws_ssm_parameter" "load_test_datasource_username" { + name = "${var.load_test_parameter_prefix}/spring.datasource.username" + type = "String" + value = local.db_root_username + overwrite = true +} + +resource "aws_ssm_parameter" "load_test_datasource_password" { + name = "${var.load_test_parameter_prefix}/spring.datasource.password" + type = "SecureString" + value = local.db_root_password + key_id = var.ssm_kms_key_id + overwrite = true + tier = "Standard" +} diff --git a/environment/load_test/output.tf b/environment/load_test/output.tf new file mode 100644 index 0000000..901044a --- /dev/null +++ b/environment/load_test/output.tf @@ -0,0 +1,94 @@ +output "load_test_rds_endpoint" { + description = "Load test RDS endpoint" + value = aws_db_instance.load_test.address +} + +output "load_test_rds_port" { + description = "Load test RDS port" + value = aws_db_instance.load_test.port +} + +output "load_test_rds_identifier" { + description = "Load test RDS identifier" + value = aws_db_instance.load_test.identifier +} + +output "load_test_db_name" { + description = "Load test database name" + value = var.db_name +} + +output "prod_rds_endpoint" { + description = "Prod RDS endpoint used as dump source" + value = data.aws_db_instance.prod.address +} + +output "prod_rds_port" { + description = "Prod RDS port" + value = data.aws_db_instance.prod.port +} + +output "prod_api_instance_id" { + description = "Prod API EC2 instance ID used to run migration commands" + value = data.aws_instance.prod_api.id +} + +output "stage_api_instance_id" { + description = "Stage API EC2 instance ID" + value = data.aws_instance.stage_api.id +} + +output "stage_api_public_ip" { + description = "Stage API EC2 public IP" + value = data.aws_instance.stage_api.public_ip +} + +output "load_test_ssm_parameter_prefix" { + description = "SSM Parameter Store prefix for load test datasource values" + value = var.load_test_parameter_prefix +} + +output "load_test_db_username_parameter_name" { + description = "SSM parameter name containing the load test DB username" + value = var.load_test_db_username_parameter_name +} + +output "load_test_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the load test DB password" + value = var.load_test_db_password_parameter_name +} + +output "prod_db_username_parameter_name" { + description = "SSM parameter name containing the prod DB username" + value = var.prod_db_username_parameter_name +} + +output "prod_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the prod DB password" + value = var.prod_db_password_parameter_name +} + +output "load_generator_instance_id" { + description = "k6 load generator EC2 instance ID" + value = aws_instance.load_generator.id +} + +output "load_generator_private_ip" { + description = "k6 load generator private IP" + value = aws_instance.load_generator.private_ip +} + +output "load_generator_k6_dir" { + description = "Directory where k6 files are placed on the load generator" + value = var.load_generator_k6_dir +} + +output "load_test_target_base_url" { + description = "Default target base URL for k6" + value = var.load_test_target_base_url +} + +output "k6_prometheus_remote_write_url" { + description = "Default Prometheus remote-write URL for k6" + value = var.k6_prometheus_remote_write_url +} diff --git a/environment/load_test/provider.tf b/environment/load_test/provider.tf index 3c3f8d1..8b41b3d 100644 --- a/environment/load_test/provider.tf +++ b/environment/load_test/provider.tf @@ -1,3 +1,22 @@ +terraform { + required_version = ">= 1.10.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "solid-connection-tfstate" + key = "env/load_test/terraform.tfstate" + region = "ap-northeast-2" + use_lockfile = true + encrypt = true + } +} + provider "aws" { region = "ap-northeast-2" default_tags { diff --git a/environment/load_test/variables.tf b/environment/load_test/variables.tf index 6f74e1f..9b2e2f6 100644 --- a/environment/load_test/variables.tf +++ b/environment/load_test/variables.tf @@ -1 +1,124 @@ -# TODO:: 부하 테스트 인스턴스용 변수 정의 +variable "rds_identifier" { + description = "RDS identifier for load test" + type = string +} + +variable "db_instance_class" { + description = "RDS instance class for load test" + type = string +} + +variable "allocated_storage" { + description = "RDS storage in GiB" + type = number + default = 20 +} + +variable "db_engine_version" { + description = "MySQL engine version" + type = string +} + +variable "db_parameter_group_name" { + description = "MySQL parameter group name" + type = string +} + +variable "db_name" { + description = "Application database name" + type = string + default = "solid_connection" +} + +variable "load_test_db_username_parameter_name" { + description = "SSM parameter name containing the load test DB root username" + type = string +} + +variable "load_test_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the load test DB root password" + type = string +} + +variable "prod_db_username_parameter_name" { + description = "SSM parameter name containing the prod DB username" + type = string + default = "/solid-connection/prod/spring.datasource.username" +} + +variable "prod_db_password_parameter_name" { + description = "SSM SecureString parameter name containing the prod DB password" + type = string + default = "/solid-connection/prod/spring.datasource.password" +} + +variable "kms_key_arn" { + description = "KMS key ARN for RDS storage encryption" + type = string +} + +variable "ssm_kms_key_id" { + description = "KMS key ID or ARN for SSM SecureString. Null uses the AWS managed aws/ssm key." + type = string + default = null + nullable = true +} + +variable "prod_rds_identifier" { + description = "Source prod RDS identifier" + type = string +} + +variable "prod_api_instance_name" { + description = "Name tag of the prod API EC2 instance used to run dump/restore" + type = string + default = "solid-connection-server-prod" +} + +variable "stage_api_instance_name" { + description = "Name tag of the stage API EC2 instance that will connect to load test RDS" + type = string + default = "solid-connection-server-stage" +} + +variable "load_test_parameter_prefix" { + description = "SSM Parameter Store prefix for load test datasource values" + type = string + default = "/solid-connection/loadtest" +} + +variable "load_generator_instance_type" { + description = "EC2 instance type for the k6 load generator" + type = string + default = "c7i.xlarge" +} + +variable "load_generator_instance_profile_name" { + description = "Existing IAM instance profile name for the k6 load generator. It must allow SSM RunCommand." + type = string + default = "solid-connection-load-test-generator" +} + +variable "load_generator_root_volume_size" { + description = "Root volume size in GiB for the k6 load generator" + type = number + default = 20 +} + +variable "load_generator_k6_dir" { + description = "Directory where k6 files are placed on the load generator" + type = string + default = "/home/ubuntu/solid-connection-load-test/k6" +} + +variable "load_test_target_base_url" { + description = "Default target base URL for k6" + type = string + default = "https://api.stage.solid-connection.com" +} + +variable "k6_prometheus_remote_write_url" { + description = "Default Prometheus remote-write URL for k6" + type = string + default = "http://132.145.83.182:9090/api/v1/write" +} diff --git a/environment/stage/main.tf b/environment/stage/main.tf index 3f3e129..a0b0b44 100644 --- a/environment/stage/main.tf +++ b/environment/stage/main.tf @@ -6,8 +6,8 @@ data "aws_vpc" "default" { module "stage_stack" { source = "../../modules/app_stack" - env_name = "stage" - vpc_id = data.aws_vpc.default.id + env_name = "stage" + vpc_id = data.aws_vpc.default.id ami_id = var.ami_id @@ -15,13 +15,13 @@ module "stage_stack" { ec2_iam_instance_profile = var.ec2_iam_instance_profile # 키페어 및 접속 허용 - key_name = var.key_name + key_name = var.key_name # 인스턴스 스펙 - instance_type = var.server_instance_type + instance_type = var.server_instance_type # RDS 미사용 (Docker container로 대체) - enable_rds = false + enable_rds = false # 보안 그룹 규칙 api_ingress_rules = var.api_ingress_rules diff --git a/modules/app_stack/ec2.tf b/modules/app_stack/ec2.tf index b49aa52..6734a28 100644 --- a/modules/app_stack/ec2.tf +++ b/modules/app_stack/ec2.tf @@ -21,6 +21,7 @@ data "cloudinit_config" "app_init" { content = file("${path.module}/../common/scripts/docker_setup.sh") filename = "1_docker_install.sh" } + } # API Server (EC2) @@ -100,9 +101,9 @@ resource "null_resource" "update_side_infra" { triggers = { script_hash = sha256(templatefile("${path.module}/scripts/side_infra_setup.sh.tftpl", { - work_dir = var.work_dir - alloy_env_name = var.alloy_env_name - alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { + work_dir = var.work_dir + alloy_env_name = var.alloy_env_name + alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { loki_ip = data.aws_instance.monitoring_server.private_ip }) redis_version = var.redis_version @@ -120,9 +121,9 @@ resource "null_resource" "update_side_infra" { provisioner "file" { content = templatefile("${path.module}/scripts/side_infra_setup.sh.tftpl", { - work_dir = var.work_dir - alloy_env_name = var.alloy_env_name - alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { + work_dir = var.work_dir + alloy_env_name = var.alloy_env_name + alloy_config_content = templatefile("${path.module}/../../config/side-infra/config.alloy.tftpl", { loki_ip = data.aws_instance.monitoring_server.private_ip }) redis_version = var.redis_version diff --git a/modules/app_stack/variables.tf b/modules/app_stack/variables.tf index 33f8b1a..1c5028d 100644 --- a/modules/app_stack/variables.tf +++ b/modules/app_stack/variables.tf @@ -67,7 +67,7 @@ variable "additional_db_users" { database = string privileges = list(string) })) - default = {} + default = {} } variable "db_engine_version" { diff --git a/scripts/load_test/README.md b/scripts/load_test/README.md new file mode 100644 index 0000000..dacc960 --- /dev/null +++ b/scripts/load_test/README.md @@ -0,0 +1,100 @@ +# 부하 테스트 자동화 + +부하 테스트용 임시 RDS와 k6 전용 EC2를 생성하고, prod RDS 데이터를 복사한 뒤 stage 서버가 loadtest datasource를 바라보도록 전환하는 자동화입니다. 시작, 실행, 종료는 GitHub Actions에서 수동으로 실행합니다. + +## secret에 필요한 값 + +EC2 기반 k6 실행을 위해 새로 `config/secrets/load_test.tfvars`에 추가해야 하는 값은 없습니다. + +기존 loadtest Terraform에 필요한 민감값만 secret에 둡니다. + +- `load_test_db_username_parameter_name` +- `load_test_db_password_parameter_name` + +아래 값들은 민감값이 아니므로 secret에 넣지 않고 Terraform 기본값이나 GitHub Actions 입력값으로 처리합니다. + +- k6 전용 EC2 instance type: 기본값 `c7i.xlarge` +- k6 전용 EC2 IAM instance profile 이름: 기본값 `solid-connection-load-test-generator` +- k6 target URL: 기본값 `https://api.stage.solid-connection.com` +- k6 VU, iterations, max duration: **Load Test Run** workflow 입력값 +- Prometheus remote-write URL: 기본값 사용 또는 **Load Test Run** workflow 입력값 + +기본 instance profile 이름을 쓰지 않을 경우 secret이 아니라 GitHub Actions variable 또는 `TF_VAR_load_generator_instance_profile_name`으로 덮어씁니다. + +## 원칙 + +- 사람이 로컬에서 `terraform apply` 또는 `terraform destroy`를 직접 실행하지 않습니다. +- stage EC2는 부하를 받는 대상이므로 stage EC2에서 k6를 실행하지 않습니다. +- k6는 loadtest Terraform이 생성한 별도 EC2에서 실행합니다. +- SSH private key는 사용하지 않습니다. stage/prod/load-generator EC2 작업은 SSM RunCommand로 실행합니다. + +## 시작 + +1. GitHub에서 **Actions > Load Test Start**를 엽니다. +2. **Run workflow**를 클릭합니다. +3. 필요한 입력값을 선택하고 실행합니다. + +입력값: + +- `switch_stage_to_loadtest`: stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +- `copy_prod_data`: prod RDS 데이터를 loadtest RDS로 복사합니다. + +workflow는 GitHub Actions runner에서 `environment/load_test`의 Terraform을 apply하고 `scripts/load_test/start.sh`를 실행합니다. + +## 시작 시 수행 작업 + +1. GitHub Actions가 `environment/load_test`에서 Terraform apply를 실행합니다. +2. loadtest RDS, 보안 그룹, k6 전용 EC2를 생성합니다. +3. loadtest datasource 값을 Parameter Store에 작성합니다. +4. prod DB와 loadtest DB 접속 정보를 Parameter Store에서 읽습니다. +5. SSM RunCommand로 prod EC2에서 `mysqldump`를 실행하고 loadtest RDS로 복원합니다. +6. SSM RunCommand로 stage 앱을 `dev,loadtest` 프로필로 재기동합니다. +7. 데이터 이관용 임시 Parameter Store 값을 삭제합니다. + +## 실행 + +1. GitHub에서 **Actions > Load Test Run**을 엽니다. +2. **Run workflow**를 클릭합니다. +3. VU, iterations, max duration을 입력하고 실행합니다. + +workflow는 `scripts/load_test/run_k6.sh`를 실행합니다. + +실행 시 수행 작업: + +1. Terraform output에서 k6 전용 EC2 ID와 기본 실행값을 읽습니다. +2. SSM RunCommand로 k6 파일을 k6 전용 EC2에 배치합니다. +3. k6 전용 EC2에서 `set_up_xk6.sh`로 k6 binary를 준비합니다. +4. k6 전용 EC2에서 `whole-user-flow.js`를 실행합니다. + +## 종료 + +1. GitHub에서 **Actions > Load Test Stop**을 엽니다. +2. **Run workflow**를 클릭합니다. +3. 필요한 입력값을 선택하고 실행합니다. + +입력값: + +- `restore_stage_dev`: stage 앱을 기존 dev compose 구성으로 되돌립니다. +- `destroy_rds`: loadtest Terraform stack을 제거합니다. + +workflow는 `scripts/load_test/stop.sh`를 실행하고, 선택값에 따라 stage 복구와 Terraform destroy를 수행합니다. Terraform destroy에는 loadtest RDS와 k6 전용 EC2 제거가 포함됩니다. + +## k6 파일 + +포함 파일: + +- `createPost.json` +- `updatePost.json` +- `whole-user-flow.js` +- `set_up_xk6.sh` +- `script/set-load-test.sh` + +이 파일들은 stage EC2가 아니라 k6 전용 EC2에 동기화됩니다. + +## 참고 + +- GitHub Actions는 OIDC로 `AWS_ROLE_ARN`을 assume합니다. +- private submodule checkout에는 `GH_PAT`를 사용합니다. +- prod/stage EC2는 `Name` tag로 조회합니다. +- prod/loadtest DB username/password는 Parameter Store에서 읽습니다. +- loadtest RDS security group은 prod/stage API EC2 security group에서 오는 MySQL 접근만 허용합니다. diff --git a/scripts/load_test/run_k6.sh b/scripts/load_test/run_k6.sh new file mode 100644 index 0000000..e2b0477 --- /dev/null +++ b/scripts/load_test/run_k6.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +set -euo pipefail + +TERRAFORM_DIR="environment/load_test" +LOCAL_K6_DIR="config/load-test/k6" +K6_SCRIPT="whole-user-flow.js" +TARGET_BASE_URL="" +PROMETHEUS_REMOTE_WRITE_URL="" +K6_VUS="10" +K6_ITERATIONS="10" +K6_MAX_DURATION="15m" +SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-3600}" + +usage() { + cat <<'EOF' +Usage: scripts/load_test/run_k6.sh [options] + +Options: + --terraform-dir PATH Default: environment/load_test + --local-k6-dir PATH Default: config/load-test/k6 + --script FILE Default: whole-user-flow.js + --target-base-url URL Default: Terraform output load_test_target_base_url + --prometheus-remote-write-url URL Default: Terraform output k6_prometheus_remote_write_url + --vus VALUE Default: 10 + --iterations VALUE Default: 10 + --max-duration VALUE Default: 15m + --ssm-command-timeout-seconds Default: 3600 + -h, --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --local-k6-dir) LOCAL_K6_DIR="$2"; shift 2 ;; + --script) K6_SCRIPT="$2"; shift 2 ;; + --target-base-url) TARGET_BASE_URL="$2"; shift 2 ;; + --prometheus-remote-write-url) PROMETHEUS_REMOTE_WRITE_URL="$2"; shift 2 ;; + --vus) K6_VUS="$2"; shift 2 ;; + --iterations) K6_ITERATIONS="$2"; shift 2 ;; + --max-duration) K6_MAX_DURATION="$2"; shift 2 ;; + --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Required command not found: $1" >&2 + exit 1 + fi +} + +require_command terraform +require_command aws +require_command jq +require_command base64 + +tf_output() { + terraform -chdir="$TERRAFORM_DIR" output -raw "$1" +} + +send_ssm_command() { + local instance_id="$1" + local comment="$2" + local commands_json="$3" + + local command_id + local started_at + command_id="$(aws ssm send-command \ + --instance-ids "$instance_id" \ + --document-name "AWS-RunShellScript" \ + --comment "$comment" \ + --parameters "$commands_json" \ + --query "Command.CommandId" \ + --output text)" + started_at="$(date +%s)" + + local status + while true; do + sleep 5 + status="$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "Status" \ + --output text 2>/dev/null || true)" + + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command timed out after ${SSM_COMMAND_TIMEOUT_SECONDS}s: $comment" >&2 + exit 1 + fi + + case "$status" in + Pending|InProgress|Delayed|"") continue ;; + Success) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "StandardOutputContent" \ + --output text || true + break + ;; + *) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command failed with status $status: $comment" >&2 + exit 1 + ;; + esac + done +} + +wait_for_ssm() { + local instance_id="$1" + local started_at + started_at="$(date +%s)" + + while true; do + local ping_status + ping_status="$(aws ssm describe-instance-information \ + --filters "Key=InstanceIds,Values=${instance_id}" \ + --query "InstanceInformationList[0].PingStatus" \ + --output text 2>/dev/null || true)" + + if [[ "$ping_status" == "Online" ]]; then + break + fi + + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + echo "SSM agent did not become online after ${SSM_COMMAND_TIMEOUT_SECONDS}s: ${instance_id}" >&2 + exit 1 + fi + + sleep 10 + done +} + +file_base64() { + base64 "$1" | tr -d '\n' +} + +sync_file() { + local instance_id="$1" + local target_dir="$2" + local relative_path="$3" + local source_path="${LOCAL_K6_DIR}/${relative_path}" + + if [[ ! -f "$source_path" ]]; then + echo "Missing k6 file: $source_path" >&2 + exit 1 + fi + + local commands_json + commands_json="$(jq -cn \ + --arg target "${target_dir}/${relative_path}" \ + --arg content "$(file_base64 "$source_path")" \ + '{ + commands: [ + "set -euo pipefail", + "mkdir -p \"$(dirname \"\($target)\")\"", + "printf %s \($content | @sh) | base64 -d > \($target | @sh)" + ] + }')" + + send_ssm_command "$instance_id" "Sync ${relative_path} to load generator" "$commands_json" +} + +terraform -chdir="$TERRAFORM_DIR" init + +load_generator_instance_id="$(tf_output load_generator_instance_id)" +load_generator_k6_dir="$(tf_output load_generator_k6_dir)" +tf_target_base_url="$(tf_output load_test_target_base_url)" +tf_prometheus_remote_write_url="$(tf_output k6_prometheus_remote_write_url)" + +TARGET_BASE_URL="${TARGET_BASE_URL:-$tf_target_base_url}" +PROMETHEUS_REMOTE_WRITE_URL="${PROMETHEUS_REMOTE_WRITE_URL:-$tf_prometheus_remote_write_url}" + +wait_for_ssm "$load_generator_instance_id" + +for relative_path in \ + "createPost.json" \ + "updatePost.json" \ + "whole-user-flow.js" \ + "set_up_xk6.sh" \ + "script/set-load-test.sh"; do + sync_file "$load_generator_instance_id" "$load_generator_k6_dir" "$relative_path" +done + +run_commands_json="$(jq -cn \ + --arg k6_dir "$load_generator_k6_dir" \ + --arg script "$K6_SCRIPT" \ + --arg target_base_url "$TARGET_BASE_URL" \ + --arg prometheus_url "$PROMETHEUS_REMOTE_WRITE_URL" \ + --arg vus "$K6_VUS" \ + --arg iterations "$K6_ITERATIONS" \ + --arg max_duration "$K6_MAX_DURATION" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($k6_dir)", + "chmod +x set_up_xk6.sh script/set-load-test.sh", + "chown -R ubuntu:ubuntu \($k6_dir)", + "if [ ! -x ./k6 ]; then sudo -u ubuntu -H ./set_up_xk6.sh; fi", + "sudo -u ubuntu -H env BASE_URL=\($target_base_url | @sh) K6_PROMETHEUS_RW_SERVER_URL=\($prometheus_url | @sh) K6_VUS=\($vus | @sh) K6_ITERATIONS=\($iterations | @sh) K6_MAX_DURATION=\($max_duration | @sh) ./k6 run \($script | @sh)" + ] + }')" + +send_ssm_command "$load_generator_instance_id" "Run k6 load test" "$run_commands_json" diff --git a/scripts/load_test/start.sh b/scripts/load_test/start.sh new file mode 100644 index 0000000..e3a7414 --- /dev/null +++ b/scripts/load_test/start.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash +set -euo pipefail + +TERRAFORM_DIR="environment/load_test" +VAR_FILE="../../config/secrets/load_test.tfvars" +DATABASE_NAME="" +MIGRATION_PARAMETER_PREFIX="/solid-connection/loadtest/migration" +PROD_DB_USERNAME_PARAMETER="" +PROD_DB_PASSWORD_PARAMETER="" +LOADTEST_DB_USERNAME_PARAMETER="" +LOADTEST_DB_PASSWORD_PARAMETER="" +SWITCH_STAGE_TO_LOADTEST="false" +STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" +STAGE_COMPOSE_FILE="docker-compose.dev.yml" +SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-1800}" +SKIP_TERRAFORM_APPLY="false" +SKIP_DATA_COPY="false" + +usage() { + cat <<'EOF' +Usage: scripts/load_test/start.sh [options] + +Options: + --terraform-dir PATH Default: environment/load_test + --var-file PATH Default: ../../config/secrets/load_test.tfvars + --prod-db-username-parameter Default: Terraform output prod_db_username_parameter_name + --prod-db-password-parameter Default: Terraform output prod_db_password_parameter_name + --loadtest-db-username-parameter Default: Terraform output load_test_db_username_parameter_name + --loadtest-db-password-parameter Default: Terraform output load_test_db_password_parameter_name + --database-name VALUE Default: Terraform output load_test_db_name + --migration-prefix VALUE Default: /solid-connection/loadtest/migration + --switch-stage-to-loadtest Restart stage app through SSM with dev,loadtest profiles + --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev + --stage-compose-file VALUE Default: docker-compose.dev.yml + --ssm-command-timeout-seconds Default: 1800 + --skip-terraform-apply + --skip-data-copy + -h, --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --var-file) VAR_FILE="$2"; shift 2 ;; + --prod-db-username-parameter) PROD_DB_USERNAME_PARAMETER="$2"; shift 2 ;; + --prod-db-password-parameter) PROD_DB_PASSWORD_PARAMETER="$2"; shift 2 ;; + --loadtest-db-username-parameter) LOADTEST_DB_USERNAME_PARAMETER="$2"; shift 2 ;; + --loadtest-db-password-parameter) LOADTEST_DB_PASSWORD_PARAMETER="$2"; shift 2 ;; + --database-name) DATABASE_NAME="$2"; shift 2 ;; + --migration-prefix) MIGRATION_PARAMETER_PREFIX="$2"; shift 2 ;; + --switch-stage-to-loadtest) SWITCH_STAGE_TO_LOADTEST="true"; shift ;; + --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; + --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; + --skip-terraform-apply) SKIP_TERRAFORM_APPLY="true"; shift ;; + --skip-data-copy) SKIP_DATA_COPY="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +require_value() { + local name="$1" + local value="$2" + if [[ -z "$value" ]]; then + echo "Missing required option: $name" >&2 + exit 1 + fi +} + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Required command not found: $1" >&2 + exit 1 + fi +} + +require_command terraform +require_command aws +require_command jq + +tf_output() { + terraform -chdir="$TERRAFORM_DIR" output -raw "$1" +} + +send_ssm_command() { + local instance_id="$1" + local comment="$2" + local commands_json="$3" + + local command_id + local started_at + command_id="$(aws ssm send-command \ + --instance-ids "$instance_id" \ + --document-name "AWS-RunShellScript" \ + --comment "$comment" \ + --parameters "$commands_json" \ + --query "Command.CommandId" \ + --output text)" + started_at="$(date +%s)" + + local status + while true; do + sleep 5 + status="$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "Status" \ + --output text 2>/dev/null || true)" + + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command timed out after ${SSM_COMMAND_TIMEOUT_SECONDS}s: $comment" >&2 + exit 1 + fi + + case "$status" in + Pending|InProgress|Delayed|"") continue ;; + Success) break ;; + *) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command failed with status $status: $comment" >&2 + exit 1 + ;; + esac + done +} + +delete_temp_parameters() { + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" >/dev/null 2>&1 || true + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" >/dev/null 2>&1 || true + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-username" >/dev/null 2>&1 || true + aws ssm delete-parameter --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-password" >/dev/null 2>&1 || true +} + +if [[ "$SKIP_TERRAFORM_APPLY" != "true" ]]; then + terraform -chdir="$TERRAFORM_DIR" init + terraform -chdir="$TERRAFORM_DIR" apply -auto-approve -var-file="$VAR_FILE" +fi + +prod_instance_id="$(tf_output prod_api_instance_id)" +stage_instance_id="$(tf_output stage_api_instance_id)" +stage_public_ip="$(tf_output stage_api_public_ip)" +prod_endpoint="$(tf_output prod_rds_endpoint)" +prod_port="$(tf_output prod_rds_port)" +loadtest_endpoint="$(tf_output load_test_rds_endpoint)" +loadtest_port="$(tf_output load_test_rds_port)" +load_generator_instance_id="$(tf_output load_generator_instance_id)" +loadtest_db_name="$(tf_output load_test_db_name)" +tf_prod_db_username_parameter="$(tf_output prod_db_username_parameter_name)" +tf_prod_db_password_parameter="$(tf_output prod_db_password_parameter_name)" +tf_loadtest_db_username_parameter="$(tf_output load_test_db_username_parameter_name)" +tf_loadtest_db_password_parameter="$(tf_output load_test_db_password_parameter_name)" + +DATABASE_NAME="${DATABASE_NAME:-$loadtest_db_name}" +PROD_DB_USERNAME_PARAMETER="${PROD_DB_USERNAME_PARAMETER:-$tf_prod_db_username_parameter}" +PROD_DB_PASSWORD_PARAMETER="${PROD_DB_PASSWORD_PARAMETER:-$tf_prod_db_password_parameter}" +LOADTEST_DB_USERNAME_PARAMETER="${LOADTEST_DB_USERNAME_PARAMETER:-$tf_loadtest_db_username_parameter}" +LOADTEST_DB_PASSWORD_PARAMETER="${LOADTEST_DB_PASSWORD_PARAMETER:-$tf_loadtest_db_password_parameter}" + +if [[ "$SKIP_DATA_COPY" != "true" ]]; then + trap delete_temp_parameters EXIT + + prod_db_username="$(aws ssm get-parameter \ + --name "$PROD_DB_USERNAME_PARAMETER" \ + --query "Parameter.Value" \ + --output text)" + + prod_db_password="$(aws ssm get-parameter \ + --name "$PROD_DB_PASSWORD_PARAMETER" \ + --with-decryption \ + --query "Parameter.Value" \ + --output text)" + + loadtest_db_username="$(aws ssm get-parameter \ + --name "$LOADTEST_DB_USERNAME_PARAMETER" \ + --query "Parameter.Value" \ + --output text)" + + loadtest_db_password="$(aws ssm get-parameter \ + --name "$LOADTEST_DB_PASSWORD_PARAMETER" \ + --with-decryption \ + --query "Parameter.Value" \ + --output text)" + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/prod-db-username" \ + --type String \ + --value "$prod_db_username" \ + --overwrite >/dev/null + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/prod-db-password" \ + --type SecureString \ + --value "$prod_db_password" \ + --overwrite >/dev/null + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-username" \ + --type String \ + --value "$loadtest_db_username" \ + --overwrite >/dev/null + + aws ssm put-parameter \ + --name "$MIGRATION_PARAMETER_PREFIX/loadtest-db-password" \ + --type SecureString \ + --value "$loadtest_db_password" \ + --overwrite >/dev/null + + copy_commands_json="$(jq -cn \ + --arg prefix "$MIGRATION_PARAMETER_PREFIX" \ + --arg prod_endpoint "$prod_endpoint" \ + --arg prod_port "$prod_port" \ + --arg loadtest_endpoint "$loadtest_endpoint" \ + --arg loadtest_port "$loadtest_port" \ + --arg database "$DATABASE_NAME" \ + '{ + commands: [ + "set -euo pipefail", + "export DEBIAN_FRONTEND=noninteractive", + "if ! command -v mysqldump >/dev/null 2>&1 || ! command -v mysql >/dev/null 2>&1; then sudo apt-get update && sudo apt-get install -y mysql-client; fi", + "PROD_USER=$(aws ssm get-parameter --name \($prefix)/prod-db-username --query Parameter.Value --output text)", + "PROD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/prod-db-password --with-decryption --query Parameter.Value --output text)", + "LOAD_USER=$(aws ssm get-parameter --name \($prefix)/loadtest-db-username --query Parameter.Value --output text)", + "LOAD_PASSWORD=$(aws ssm get-parameter --name \($prefix)/loadtest-db-password --with-decryption --query Parameter.Value --output text)", + "DUMP_FILE=/tmp/solid-connection-loadtest-$(date +%Y%m%d%H%M%S).sql.gz", + "trap '\''rm -f \"$DUMP_FILE\"'\'' EXIT", + "MYSQL_PWD=\"$PROD_PASSWORD\" mysqldump --single-transaction --set-gtid-purged=OFF --column-statistics=0 -h \($prod_endpoint) -P \($prod_port) -u \"$PROD_USER\" \($database) | gzip > \"$DUMP_FILE\"", + "MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" -e \"DROP DATABASE IF EXISTS \\\`\($database)\\\`; CREATE DATABASE \\\`\($database)\\\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;\"", + "gunzip -c \"$DUMP_FILE\" | MYSQL_PWD=\"$LOAD_PASSWORD\" mysql -h \($loadtest_endpoint) -P \($loadtest_port) -u \"$LOAD_USER\" \($database)", + "rm -f \"$DUMP_FILE\"" + ] + }')" + + send_ssm_command "$prod_instance_id" "Copy prod RDS data to load test RDS" "$copy_commands_json" +fi + +if [[ "$SWITCH_STAGE_TO_LOADTEST" == "true" ]]; then + stage_commands_json="$(jq -cn \ + --arg app_dir "$STAGE_APP_DIR" \ + --arg compose_file "$STAGE_COMPOSE_FILE" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($app_dir)", + "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", + "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", + "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", + "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", + "cat > docker-compose.loadtest.override.yml <<'\''YAML'\''\nservices:\n solid-connection-dev:\n environment:\n - SPRING_PROFILES_ACTIVE=dev,loadtest\n - AWS_REGION=ap-northeast-2\n - SPRING_DATA_REDIS_HOST=127.0.0.1\n - SPRING_DATA_REDIS_PORT=6379\nYAML", + "docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml down || true", + "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) -f docker-compose.loadtest.override.yml up -d solid-connection-dev" + ] + }')" + + send_ssm_command "$stage_instance_id" "Switch stage app to load test datasource" "$stage_commands_json" +fi + +echo "Load test environment is ready." +echo "RDS endpoint: ${loadtest_endpoint}:${loadtest_port}" +echo "Load generator instance: ${load_generator_instance_id}" +echo "Stage instance: ${stage_instance_id}" +echo "Stage public IP: ${stage_public_ip}" diff --git a/scripts/load_test/stop.sh b/scripts/load_test/stop.sh new file mode 100644 index 0000000..005a61c --- /dev/null +++ b/scripts/load_test/stop.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -euo pipefail + +TERRAFORM_DIR="environment/load_test" +VAR_FILE="../../config/secrets/load_test.tfvars" +RESTORE_STAGE_DEV="false" +STAGE_APP_DIR="/home/ubuntu/solid-connection-dev" +STAGE_COMPOSE_FILE="docker-compose.dev.yml" +SSM_COMMAND_TIMEOUT_SECONDS="${SSM_COMMAND_TIMEOUT_SECONDS:-900}" +SKIP_TERRAFORM_DESTROY="false" + +usage() { + cat <<'EOF' +Usage: scripts/load_test/stop.sh [options] + +Options: + --terraform-dir PATH Default: environment/load_test + --var-file PATH Default: ../../config/secrets/load_test.tfvars + --restore-stage-dev Restart stage app through SSM with dev profile + --stage-app-dir PATH Default: /home/ubuntu/solid-connection-dev + --stage-compose-file VALUE Default: docker-compose.dev.yml + --ssm-command-timeout-seconds Default: 900 + --skip-terraform-destroy + -h, --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-dir) TERRAFORM_DIR="$2"; shift 2 ;; + --var-file) VAR_FILE="$2"; shift 2 ;; + --restore-stage-dev) RESTORE_STAGE_DEV="true"; shift ;; + --stage-app-dir) STAGE_APP_DIR="$2"; shift 2 ;; + --stage-compose-file) STAGE_COMPOSE_FILE="$2"; shift 2 ;; + --ssm-command-timeout-seconds) SSM_COMMAND_TIMEOUT_SECONDS="$2"; shift 2 ;; + --skip-terraform-destroy) SKIP_TERRAFORM_DESTROY="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Required command not found: $1" >&2 + exit 1 + fi +} + +require_command terraform +require_command aws +require_command jq + +tf_output() { + terraform -chdir="$TERRAFORM_DIR" output -raw "$1" +} + +send_ssm_command() { + local instance_id="$1" + local comment="$2" + local commands_json="$3" + + local command_id + local started_at + command_id="$(aws ssm send-command \ + --instance-ids "$instance_id" \ + --document-name "AWS-RunShellScript" \ + --comment "$comment" \ + --parameters "$commands_json" \ + --query "Command.CommandId" \ + --output text)" + started_at="$(date +%s)" + + local status + while true; do + sleep 5 + status="$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query "Status" \ + --output text 2>/dev/null || true)" + + if (( $(date +%s) - started_at > SSM_COMMAND_TIMEOUT_SECONDS )); then + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command timed out after ${SSM_COMMAND_TIMEOUT_SECONDS}s: $comment" >&2 + exit 1 + fi + + case "$status" in + Pending|InProgress|Delayed|"") continue ;; + Success) break ;; + *) + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --output json || true + echo "SSM command failed with status $status: $comment" >&2 + exit 1 + ;; + esac + done +} + +terraform -chdir="$TERRAFORM_DIR" init + +if [[ "$RESTORE_STAGE_DEV" == "true" ]]; then + stage_instance_id="$(tf_output stage_api_instance_id)" + + stage_commands_json="$(jq -cn \ + --arg app_dir "$STAGE_APP_DIR" \ + --arg compose_file "$STAGE_COMPOSE_FILE" \ + '{ + commands: [ + "set -euo pipefail", + "cd \($app_dir)", + "CURRENT_IMAGE=$(docker inspect -f '\''{{.Config.Image}}'\'' solid-connection-dev 2>/dev/null || true)", + "if [ -z \"$CURRENT_IMAGE\" ]; then echo \"solid-connection-dev container is not running; cannot infer image tag\" >&2; exit 1; fi", + "OWNER_LOWERCASE=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#^ghcr.io/([^/]+)/.*#\\1#'\'')", + "IMAGE_TAG=$(echo \"$CURRENT_IMAGE\" | sed -E '\''s#.*:([^:]+)$#\\1#'\'')", + "rm -f docker-compose.loadtest.override.yml", + "docker compose -f \($compose_file) down || true", + "OWNER_LOWERCASE=\"$OWNER_LOWERCASE\" IMAGE_TAG=\"$IMAGE_TAG\" docker compose -f \($compose_file) up -d" + ] + }')" + + send_ssm_command "$stage_instance_id" "Restore stage app to dev datasource" "$stage_commands_json" +fi + +if [[ "$SKIP_TERRAFORM_DESTROY" != "true" ]]; then + terraform -chdir="$TERRAFORM_DIR" destroy -auto-approve -var-file="$VAR_FILE" +fi + +echo "Load test environment has been stopped."