add compatibility tests

Author: ewhauser

test.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+
+test_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/tools
+
+. "${test_dir}"/test_compatibility.sh

tools/test_compatibility.sh

@@ -0,0 +1,21 @@
+dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+. "${dir}"/test_runner.sh
+. "${dir}"/test_helper.sh
+
+runner=$(get_test_runner "${1:-local}")
+
+test_single_file_change() {
+  run_compat_test de48add f0d09c4
+}
+
+test_workspace_change() {
+  run_compat_test 50efa15 60b40a3
+}
+
+test_multiple_file_change() {
+  run_compat_test b013593 95b4f75
+}
+
+$runner test_single_file_change
+$runner test_workspace_change
+$runner test_multiple_file_change

tools/test_helper.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env
+set -eo pipefail
+
+function run_compat_test() {
+    local start_commit="$1"
+    local end_commit="$2"
+
+    tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
+    repo_path="$tmp_dir/bazel-remote"
+    bazel_diff="$(bazel info bazel-bin)/tools/bazel-diff/bazel-diff"
+    bazel_differ="$(bazel info bazel-bin)/cli/bazel-differ_/bazel-differ"
+
+    git clone https://github.com/buchgr/bazel-remote.git "$repo_path"
+
+    common_opts="-w $repo_path -b $(which bazelisk)"
+
+    pushd "$repo_path"
+    git checkout "$start_commit"
+
+    "$bazel_diff" generate-hashes $common_opts $tmp_dir/bazel-diff-starting.txt
+    "$bazel_differ" generate-hashes $common_opts $tmp_dir/bazel-differ-starting.txt
+
+    git checkout "$end_commit"
+
+    "$bazel_diff" generate-hashes $common_opts $tmp_dir/bazel-diff-ending.txt
+    "$bazel_differ" generate-hashes $common_opts $tmp_dir/bazel-differ-ending.txt
+
+    "$bazel_diff" $common_opts -sh $tmp_dir/bazel-diff-starting.txt -fh $tmp_dir/bazel-diff-ending.txt -o $tmp_dir/bazel-diff-targets.txt
+    "$bazel_differ" diff $common_opts -s $tmp_dir/bazel-differ-starting.txt -f $tmp_dir/bazel-differ-ending.txt -o $tmp_dir/bazel-differ-targets.txt
+
+    cat $tmp_dir/bazel-diff-targets.txt | sort > $tmp_dir/bazel-diff-targets-sorted.txt
+    cat $tmp_dir/bazel-differ-targets.txt | sort > $tmp_dir/bazel-differ-targets-sorted.txt
+
+    diff $tmp_dir/bazel-diff-targets-sorted.txt $tmp_dir/bazel-differ-targets-sorted.txt
+
+    popd
+
+    rm -rf "$tmp_dir"
+}

tools/test_runner.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+#
+# Test runner functions for bazel-mypy-integration integration tests.
+# Credit: https://github.com/bazelbuild/rules_scala/blob/master/test/shell/test_runner.sh
+
+NC='\033[0m'
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+TIMOUT=60
+
+run_test_ci() {
+  # spawns the test to new process
+  local TEST_ARG=$@
+  local log_file=output_$$.log
+  echo "running test $TEST_ARG"
+  eval $TEST_ARG &>$log_file &
+  local test_pid=$!
+  SECONDS=0
+  test_pulse_printer $! $TIMOUT $TEST_ARG &
+  local pulse_printer_pid=$!
+  local result
+
+  {
+    wait $test_pid 2>/dev/null
+    result=$?
+    kill $pulse_printer_pid && wait $pulse_printer_pid 2>/dev/null || true
+  } || return 1
+
+  DURATION=$SECONDS
+  if [ $result -eq 0 ]; then
+    echo -e "\n${GREEN}Test \"$TEST_ARG\" successful ($DURATION sec) $NC"
+  else
+    echo -e "\nLog:\n"
+    cat $log_file
+    echo -e "\n${RED}Test \"$TEST_ARG\" failed $NC ($DURATION sec) $NC"
+  fi
+  return $result
+}
+
+test_pulse_printer() {
+  # makes sure something is printed to stdout while test is running
+  local test_pid=$1
+  shift
+  local timeout=$1 # in minutes
+  shift
+  local count=0
+
+  # clear the line
+  echo -e "\n"
+
+  while [ $count -lt $timeout ]; do
+    count=$(($count + 1))
+    echo -ne "Still running: \"$@\"\r"
+    sleep 60
+  done
+
+  echo -e "\n${RED}Timeout (${timeout} minutes) reached. Terminating \"$@\"${NC}\n"
+  kill -9 $test_pid
+}
+
+run_test_local() {
+  # runs the tests locally
+  set +e
+  SECONDS=0
+  TEST_ARG=$@
+  echo "running test $TEST_ARG"
+  RES=$($TEST_ARG 2>&1)
+  RESPONSE_CODE=$?
+  DURATION=$SECONDS
+  if [ $RESPONSE_CODE -eq 0 ]; then
+    echo -e "${GREEN} Test \"$TEST_ARG\" successful ($DURATION sec) $NC"
+  else
+    echo -e "\nLog:\n"
+    echo "$RES"
+    echo -e "${RED} Test \"$TEST_ARG\" failed $NC ($DURATION sec) $NC"
+    exit $RESPONSE_CODE
+  fi
+}
+
+get_test_runner() {
+  test_env=$1
+  if [[ "${test_env}" != "ci" && "${test_env}" != "local" ]]; then
+    echo -e "${RED}test_env must be either 'local' or 'ci'"
+    exit 1
+  fi
+  echo "run_test_${test_env}"
+}