Skip to content

Commit 840bb5b

Browse files
Tuseeq1Elias Kassell
Tuseeq1
and
Elias Kassell
authored
MOSS Cap1 (#1833)
* MOSSC1: Build and publish with GCB * Update both packages * Revert "fix: dry run fails when table/view/assertion already does not exsist …" (#1834) This reverts commit 4d20929. * Fixes and explanations * Fixes and explanations * Cloudbuild.yaml for old trigger * Cloudbuild * resolve comments. * Convert cloudbuild to bash scripts * newline * GH PR assign to dataform-co/dataform-reviewers * Local test cap * New line * Script fix * script fix * rename test scripts --------- Co-authored-by: Elias Kassell <elias@dataform.co>
1 parent 86254eb commit 840bb5b

11 files changed

Lines changed: 1432 additions & 1591 deletions

‎cloudbuild-publish.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
steps:
2+
- name: 'gcr.io/cloud-builders/gcloud'
3+
script: |
4+
#!/usr/bin/env bash
5+
REPO_TOKEN="$(gcloud auth print-access-token)" NPM_TOKEN=$NPM_TOKEN ./scripts/create_npmrc
6+
secretEnv: ['NPM_TOKEN']
7+
- name: gcr.io/cloud-builders/bazel:5.4.0
8+
script: |
9+
#!/usr/bin/env bash
10+
./scripts/publish
11+
availableSecrets:
12+
secretManager:
13+
- versionName: projects/178487900909/secrets/npm-publish-token/versions/2
14+
env: 'NPM_TOKEN'
15+
artifacts:
16+
npmPackages:
17+
- repository: 'https://us-central1-npm.pkg.dev/dataform-open-source/dataform-open-source'
18+
packagePath: './bazel-bin/packages/@dataform/cli'
19+
- repository: 'https://us-central1-npm.pkg.dev/dataform-open-source/dataform-open-source'
20+
packagePath: './bazel-bin/packages/@dataform/core'
21+
options:
22+
machineType: E2_HIGHCPU_8
23+
requestedVerifyOption: VERIFIED
24+
timeout: 3600s

‎cloudbuild-test.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
steps:
2+
- name: 'gcr.io/cloud-builders/gcloud'
3+
script: |
4+
#!/usr/bin/env bash
5+
REPO_TOKEN="$(gcloud auth print-access-token)" NPM_TOKEN="test_only" ./scripts/create_npmrc
6+
- name: gcr.io/cloud-builders/bazel:5.4.0
7+
script: |
8+
#!/usr/bin/env bash
9+
./scripts/run_tests_on_cloudbuild
10+
options:
11+
machineType: E2_HIGHCPU_8
12+
timeout: 3600s

‎cloudbuild-version.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
steps:
2+
# We have github personal access token stored in `github-token-access` secret in our GCP project.
3+
# This step downloads it and store it in token.txt file for later steps to use for authentication.
4+
- name: gcr.io/cloud-builders/gcloud
5+
entrypoint: 'bash'
6+
args: [ '-c', "gcloud secrets versions access latest --secret=github-token-access --format='get(payload.data)' | tr '_-' '/+' | base64 -d > token.txt" ]
7+
- name: gcr.io/cloud-builders/git
8+
entrypoint: 'bash'
9+
args:
10+
- '-c'
11+
- |
12+
_GITHUB_USER=$_GITHUB_USER _GITHUB_EMAIL=$_GITHUB_EMAIL ./scripts/create_gh_pr
13+
- name: 'gcr.io/$PROJECT_ID/github'
14+
entrypoint: 'bash'
15+
args:
16+
- '-c'
17+
- |
18+
set -e
19+
echo "Create PR..."
20+
gh auth login --with-token < token.txt
21+
gh pr create -t "Publish a new NPM versions" -b "Updating NPM package version to $(cat version.bzl | grep DF_VERSION | awk '{ print $3 }' | sed "s/\"//g")" -B $BRANCH_NAME -H $(cat git_branch_name.txt) -r dataform-co/dataform-reviewers
22+
options:
23+
automapSubstitutions: true

‎cloudbuild.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
steps:
2+
- name: 'gcr.io/cloud-builders/gcloud'
3+
script: |
4+
#!/usr/bin/env bash
5+
REPO_TOKEN="$(gcloud auth print-access-token)" NPM_TOKEN="test_only" ./scripts/create_npmrc
26
- name: gcr.io/cloud-builders/bazel:5.4.0
3-
entrypoint: bash
4-
args:
5-
- ./scripts/run_tests
7+
script: |
8+
#!/usr/bin/env bash
9+
./scripts/run_tests_on_cloudbuild
610
options:
711
machineType: E2_HIGHCPU_8
812
timeout: 3600s

‎scripts/create_gh_pr‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
set -e
2+
3+
echo "Set the git config..."
4+
git config user.name $_GITHUB_USER
5+
git config user.email $_GITHUB_EMAIL
6+
git remote set-url origin https://$_GITHUB_USER:$(cat token.txt)@github.com/dataform-co/dataform.git
7+
8+
export git_branch_name=npm_veriosn_$(cat version.bzl | grep DF_VERSION | awk '{ print $3 }' | sed "s/\"//g")
9+
10+
echo "Create new branch $git_branch_name..."
11+
git checkout -b $git_branch_name
12+
13+
echo "Update version..."
14+
./scripts/update_version
15+
16+
git add version.bzl
17+
git commit -m "Update the npm package version"
18+
19+
echo "Push changes to remote..."
20+
git push origin $git_branch_name
21+
echo $git_branch_name > git_branch_name.txt

‎scripts/create_npmrc‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
set -e
2+
cat > ~/.npmrc <<EOL
3+
//registry.npmjs.org/:_authToken=$NPM_TOKEN
4+
registry=https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/
5+
//us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/:always-auth=true
6+
//us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/:_authToken=${REPO_TOKEN}
7+
EOL

‎scripts/publish‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ if [ "$(git status --porcelain)" ]; then
1010
fi
1111

1212
# Run all the tests.
13-
13+
bazel run @nodejs//:yarn config set registry https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/
1414
bazel test //... --build_tests_only
1515

1616
VERSION=$(cat version.bzl | grep DF_VERSION | awk '{ print $3 }' | sed "s/\"//g")

‎scripts/run_tests‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
#!/bin/bash
22
set -e
33

4-
# Run tslint.
5-
bazel run @nodejs//:yarn
6-
bazel build @npm//tslint/bin:tslint && bazel-bin/external/npm/tslint/bin/tslint.sh --project .
4+
# Set env.
5+
REPO_TOKEN="$(gcloud auth print-access-token)" NPM_TOKEN="test_only" ./scripts/create_npmrc
76

8-
# Run all the tests
9-
bazel test ... --build_tests_only --test_env=USE_CLOUD_BUILD_NETWORK=true
7+
# run tests
8+
./scripts/run_tests_on_cloudbuild

‎scripts/run_tests_on_cloudbuild‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
set -e
3+
4+
bazel run @nodejs//:yarn config set registry https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/
5+
bazel run @nodejs//:yarn
6+
7+
# Run tslint.
8+
bazel build @npm//tslint/bin:tslint && bazel-bin/external/npm/tslint/bin/tslint.sh --project .
9+
10+
# Run all the tests
11+
bazel test ... --build_tests_only --test_env=USE_CLOUD_BUILD_NETWORK=true

‎scripts/update_version‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Path to the versions.bzl file.
5+
file_path="version.bzl"
6+
7+
# Extract the current version from the file.
8+
current_version=$(grep -oP '(?<=DF_VERSION = ")[^"]+' "$file_path")
9+
10+
# Split the version into its components.
11+
IFS='.-' read -r major minor patch label <<< "$current_version"
12+
13+
# Increment the patch version.
14+
((patch++))
15+
16+
# If there's a label, add a hyphen before it.
17+
if [[ -z "$label" ]]; then
18+
label=""
19+
else
20+
label="-${label}" # Add hyphen before the label.
21+
fi
22+
23+
# Construct the new version string.
24+
new_version="$major.$minor.$patch$label"
25+
26+
# Update the file with the new version.
27+
sed -i "s/DF_VERSION = \"[^\"]*\"/DF_VERSION = \"$new_version\"/" "$file_path"
28+
29+
echo "Version updated to: $new_version"

0 commit comments

Comments
 (0)