adding get-targets command (#2)

* adding get-targets command

* debug

* explicitly build bazel diff

* add differential build example

Author: ewhauser

.github/workflows/bazel-ci.yaml

@@ -22,11 +22,32 @@ jobs:
             ~/.cache/bazel
           key: ${{ runner.os }}-${{ env.cache-name }}
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      # You won't do this in your build since you will likely use a prebuilt bazel-differ binary, but we
+      # need to bootstrap
+      - name: build
+        run: bazel build //cli:bazel-differ
+      # This section starts an example of how to use get-targets in your CI process
+      - name: Get revisions
+        id: get-revisions
+        run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN  }} ./tools/diff-sha.sh
+      - name: get-test-targets
+        run: $(bazel info bazel-bin)/cli/bazel-differ_/bazel-differ get-targets -w $(pwd) -b $(which bazelisk) -s ${{ steps.get-revisions.outputs.previous_sha }} -f ${{ steps.get-revisions.outputs.current_sha }} -o test_targets.txt
+      - name: run-test-targets
+        run: |
+          cat test_targets.txt
+          bazel test --target_pattern_file=test_targets.txt
+      - name: get-build-targets
+        run: $(bazel info bazel-bin)/cli/bazel-differ_/bazel-differ get-targets -w $(pwd) -b $(which bazelisk) -s ${{ steps.get-revisions.outputs.previous_sha }} -f ${{ steps.get-revisions.outputs.current_sha }} -o build_targets.txt
+      - name: run-build-targets
+        run: |
+          cat build_targets.txt
+          bazel build --target_pattern_file=build_targets.txt
+      # Run tests
       - name: gazelle check
         run: bazel run //:gazelle_ci
-      - name: test
-        run: bazel test //...
-      - name: build
-        run: bazel build //...
+      - name: build bazel-diff
+        run: bazel build //tools/bazel-diff/...
       - name: compatibility tests
         run: ./test.sh

.gitignore

@@ -1,3 +1 @@
 .idea
-bazel-diff
-*.txt

README.md

@@ -2,14 +2,79 @@
 
 `bazel-differ` is a command line interface for Bazel that helps you do incremental builds across different Git versions. `bazel-differ` is a mostly a pure Go port of the excellent [bazel-diff](https://github.com/tinder/bazel-diff) and should be able to function as a drop-in replacement for most use cases.
 
+## CLI
+
+```console
+bazel-differ is a CLI tool to assist with doing differential Bazel builds
+
+Usage:
+  bazel-differ [command]
+
+Available Commands:
+  completion      Generate the autocompletion script for the specified shell
+  diff            Writes to a file the impacted targets between two Bazel graph JSON files
+  generate-hashes Writes to a file the SHA256 hashes for each Bazel Target in the provided workspace.
+  get-targets     Collects a set of a targets between the specified commit ranges
+  help            Help about any command
+
+Flags:
+  -z, --bazelCommandOptions string   Additional space separated Bazel command options used when invoking Bazel
+  -b, --bazelPath string             Path to Bazel binary
+  -y, --bazelStartupOptions string   Additional space separated Bazel client startup options used when invoking Bazel
+  -h, --help                         help for bazel-differ
+  -k, --keep_going bazel query       This flag controls if bazel query will be executed with the `--keep_going` flag or not. Disabling this flag allows you to catch configuration issues in your Bazel graph, but may not work for some Bazel setups. Defaults to `true` (default true)
+  -v, --verbose                      enables verbose output
+  -w, --workspacePath string         Path to Bazel workspace directory.
+
+Use "bazel-differ [command] --help" for more information about a command.
+```
+
+If you are using as a replacement for `bazel-diff`, then you will probably use the `generate-hashes` and `diff` commands. Alternatively, you can make use of the `get-targets` command to eliminate the need for an external shell script:
+
+```console
+Collects a set of a targets between the specified commit ranges. By default,
+the final set of targets is run through a Bazel query "set({{.Targets}})" which can be customized by the -q parameter.
+If you want to query all tests impacted for the given commit range, you can do:
+
+$ bazel-differ get-targets -w path/to/workspace -b $(which bazel) -s START_HASH -f FINAL_HASH -q 'kind(".*_test",set({{.Targets}}))' -o test_targets.txt
+
+Usage:
+  bazel-differ get-targets [flags]
+
+Flags:
+      --cache-dir string          Directory to cache hashes associated with commits
+  -f, --finalRevision string      The Git revision to use to generate the ending hashes
+  -h, --help                      help for get-targets
+      --nocache                   Disables hash caching (default false)
+  -o, --output string             Filepath to write the impacted Bazel targets to, newline separated
+  -q, --query string              The query template to use when querying for changed targets
+  -s, --startingRevision string   The Git revision to generate the ending hashes
+
+Global Flags:
+  -z, --bazelCommandOptions string   Additional space separated Bazel command options used when invoking Bazel
+  -b, --bazelPath string             Path to Bazel binary
+  -y, --bazelStartupOptions string   Additional space separated Bazel client startup options used when invoking Bazel
+  -k, --keep_going bazel query       This flag controls if bazel query will be executed with the `--keep_going` flag or not. Disabling this flag allows you to catch configuration issues in your Bazel graph, but may not work for some Bazel setups. Defaults to `true` (default true)
+  -v, --verbose                      enables verbose output
+  -w, --workspacePath string         Path to Bazel workspace directory.
+```
+
+Running the example above, then allows you to run:
+
+```console
+$ bazel test --target_pattern_file=test_targets.txt
+```
+
+By default, the hashes for a given Git revision are cached to disk so you can run multiple invocations of `get-target` without paying a penalty for recalculating the hashes every time.
+
 # FAQ
 
 1. Why did you port `bazel-diff`?
 
 A couple reasons:
 
-*  The projects I work with don't have any JVM dependencies so using `bazel-diff` was adding some additional build time I wanted to eliminate.
-* With the exception of the Bazel server itself, much of the tooling in the Bazel ecosystem is written in Go.
+* The projects I work with don't have any JVM dependencies so using `bazel-diff` was adding some additional build time we wanted to eliminate
+* We wanted to add a fair amount of functionality on top of `bazel-diff` and porting it to Go wasn't a huge lift. Most tooling in the Bazel ecosystem is written in Go (with the exception of the Bazel server)
 
 2. What are the differences from `bazel-diff`?
 

cmd/BUILD.bazel

@@ -5,12 +5,14 @@ go_library(
     srcs = [
         "diff.go",
         "generate_hashes.go",
+        "get_targets.go",
         "root.go",
     ],
     importpath = "github.com/ewhauser/bazel-differ/cmd",
     visibility = ["//visibility:public"],
     deps = [
         "//internal",
+        "//internal/cache",
         "@com_github_spf13_cobra//:cobra",
     ],
 )

cmd/diff.go

@@ -1,14 +1,10 @@
 package cmd
 
 import (
-	"bufio"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"github.com/ewhauser/bazel-differ/internal"
 	"github.com/spf13/cobra"
-	"io/ioutil"
-	"log"
 	"os"
 )
 
@@ -41,46 +37,23 @@ var diffCmd = &cobra.Command{
 		targetHasher := internal.NewTargetHashingClient(GetBazelClient(), internal.Filesystem,
 			internal.NewRuleProvider())
 
-		startingHashes := readHashFile(StartingHashes)
-		finalHashes := readHashFile(FinalHashes)
+		startingHashes, err := internal.ReadHashFile(StartingHashes)
+		ExitIfError(err, "")
+		finalHashes, err := internal.ReadHashFile(FinalHashes)
+		ExitIfError(err, "")
 		targets, err := targetHasher.GetImpactedTargets(startingHashes, finalHashes)
-		if err != nil {
-			panic(err)
-		}
-		writeFile(targets, Output)
+		ExitIfError(err, "")
+		internal.WriteTargetsFile(targets, Output)
 	},
 }
 
-func writeFile(targets map[string]bool, output string) {
-	file, err := os.OpenFile(output, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
-
-	if err != nil {
-		log.Fatalf("failed creating file: %s", err)
-	}
-
-	defer file.Close()
-
-	datawriter := bufio.NewWriter(file)
-	defer datawriter.Flush()
-
-	for k := range targets {
-		_, _ = datawriter.WriteString(k + "\n")
-	}
-}
-
-func readHashFile(filename string) map[string]string {
-	x := map[string]string{}
-	startingContent, err := ioutil.ReadFile(filename)
-	if err != nil {
-		panic(err)
-	}
-	err = json.Unmarshal(startingContent, &x)
-	if err != nil {
-		panic(err)
-	}
-	return x
-}
-
 func init() {
 	rootCmd.AddCommand(diffCmd)
+	diffCmd.PersistentFlags().StringVarP(&StartingHashes, "startingHashes", "s", "",
+		"The path to the JSON file of target hashes for the initial revision. Run 'generate-hashes' to get this value.")
+	diffCmd.PersistentFlags().StringVarP(&FinalHashes, "finalHashes", "f", "",
+		"The path to the JSON file of target hashes for the final revision. Run 'generate-hashes' to get this value.")
+	diffCmd.PersistentFlags().StringVarP(&Output, "output", "o", "",
+		"Filepath to write the impacted Bazel targets to, "+
+			"newline separated")
 }

cmd/generate_hashes.go

@@ -2,19 +2,15 @@ package cmd
 
 import (
 	"bufio"
-	"bytes"
-	"encoding/json"
 	"fmt"
 	"github.com/ewhauser/bazel-differ/internal"
-	"io"
-	"io/ioutil"
 	"os"
 
 	"github.com/spf13/cobra"
 )
 
 var SeedFilepaths string
-var DisplayElapsedTime bool
+var displayElapsedTime bool
 
 // generateHashesCmd represents the generateHashes command
 var generateHashesCmd = &cobra.Command{
@@ -32,30 +28,18 @@ var generateHashesCmd = &cobra.Command{
 		}
 
 		hashes, err := targetHasher.HashAllBazelTargetsAndSourcefiles(seedfilePaths)
-		if err != nil {
-			panic(err)
-		}
-
-		var buffer bytes.Buffer
-		err = prettyEncode(hashes, &buffer)
-		if err != nil {
-			panic(err)
-		}
+		ExitIfError(err, "")
 
-		err = ioutil.WriteFile(args[0], buffer.Bytes(), 0644)
-		if err != nil {
-			panic(err)
-		}
-		fmt.Println(buffer.String())
+		res, err := internal.WriteHashFile(args[0], hashes)
+		ExitIfError(err, "")
+		fmt.Println(res)
 	},
 }
 
 func readSeedFile() map[string]bool {
 	readFile, err := os.Open(SeedFilepaths)
 
-	if err != nil {
-		panic(err)
-	}
+	ExitIfError(err, fmt.Sprintf("Error reading seedfile: %s", SeedFilepaths))
 
 	defer readFile.Close()
 
@@ -68,20 +52,15 @@ func readSeedFile() map[string]bool {
 	return seedFilepaths
 }
 
-func prettyEncode(data interface{}, out io.Writer) error {
-	enc := json.NewEncoder(out)
-	enc.SetIndent("", "  ")
-	if err := enc.Encode(data); err != nil {
-		return err
-	}
-	return nil
-}
-
 func init() {
 	rootCmd.AddCommand(generateHashesCmd)
 	generateHashesCmd.Flags().StringVarP(&SeedFilepaths, "seed-filepaths", "", "",
 		"A text file containing a newline separated list of filepaths, "+
 			"each of these filepaths will be read and used as a seed for all targets.")
-	generateHashesCmd.Flags().BoolVarP(&DisplayElapsedTime, "displayElapsedTime", "d", false,
+	generateHashesCmd.Flags().BoolVarP(&displayElapsedTime, "displayElapsedTime", "d", false,
 		"This flag controls whether to print out elapsed time for bazel query and content hashing")
+	generateHashesCmd.PersistentFlags().StringVarP(&StartingHashes, "startingHashes", "s", "",
+		"The path to the JSON file of target hashes for the initial revision. Run 'generate-hashes' to get this value.")
+	generateHashesCmd.PersistentFlags().StringVarP(&FinalHashes, "finalHashes", "f", "",
+		"The path to the JSON file of target hashes for the final revision. Run 'generate-hashes' to get this value.")
 }

cmd/get_targets.go

@@ -0,0 +1,97 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/ewhauser/bazel-differ/internal"
+	"github.com/ewhauser/bazel-differ/internal/cache"
+	"github.com/spf13/cobra"
+)
+
+var startingRevision string
+var finalRevision string
+var query string
+var cacheDir string
+var cacheDisabled bool
+
+// getTargets represents the get-targets command
+var getTargets = &cobra.Command{
+	Use:   "get-targets",
+	Short: "Collects a set of a targets between the specified commit ranges",
+	Long: `Collects a set of a targets between the specified commit ranges. By default, 
+the final set of targets is run through a Bazel query "set({{.Targets}})" which can be customized by the -q parameter. 
+If you want to query all tests impacted for the given commit range, you can do:
+
+$ bazel-differ get-targets -w path/to/workspace -b $(which bazel) -s START_HASH -f FINAL_HASH -q 'kind(".*_test",set({{.Targets}}))'" -o test_targets.txt
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		gitClient := internal.NewGitClient(WorkspacePath)
+		bazelClient := GetBazelClient()
+		cacheManager, err := cache.NewHashCacheManager(!cacheDisabled, cacheDir)
+		ExitIfError(err, "")
+		targetHasher := internal.NewTargetHashingClient(GetBazelClient(), internal.Filesystem,
+			internal.NewRuleProvider())
+
+		startingHashes := getHashes(startingRevision, gitClient, cacheManager, targetHasher)
+		endingHashes := getHashes(finalRevision, gitClient, cacheManager, targetHasher)
+
+		targets, err := targetHasher.GetImpactedTargets(startingHashes, endingHashes)
+		ExitIfError(err, "")
+
+		queriedTargets, err := bazelClient.QueryTarget(query, targets)
+		ExitIfError(err, "")
+		targetNames := targetHasher.GetNames(queriedTargets)
+
+		if Output != "" {
+			internal.WriteTargetsFile(targetNames, Output)
+		} else {
+			for k := range targetNames {
+				fmt.Println(k)
+			}
+		}
+	},
+}
+
+func getHashes(revision string, gitClient internal.GitClient, cacheManager cache.HashCacheManager,
+	targetHasher internal.TargetHashingClient) map[string]string {
+	err := gitClient.Checkout(revision)
+	ExitIfError(err, fmt.Sprintf("Unable to checkout revision: %s", revision))
+
+	var seedfilePaths = make(map[string]bool)
+	if SeedFilepaths != "" {
+		seedfilePaths = readSeedFile()
+	}
+
+	hashes, err := cacheManager.Get(context.Background(), revision)
+	ExitIfError(err, fmt.Sprintf("Error retrieving hashes for revision %s from cache", revision))
+
+	if hashes == nil {
+		hashes, err = targetHasher.HashAllBazelTargetsAndSourcefiles(seedfilePaths)
+	}
+	ExitIfError(err, "")
+
+	err = cacheManager.Put(context.Background(), revision, hashes)
+	ExitIfError(err, "")
+
+	return hashes
+}
+
+func init() {
+	rootCmd.AddCommand(getTargets)
+	getTargets.PersistentFlags().StringVarP(&startingRevision, "startingRevision", "s", "",
+		"The Git revision to generate the ending hashes")
+	getTargets.PersistentFlags().StringVarP(&finalRevision, "finalRevision", "f", "",
+		"The Git revision to use to generate the ending hashes")
+	getTargets.Flags().StringVarP(&SeedFilepaths, "seed-filepaths", "", "",
+		"A text file containing a newline separated list of filepaths, "+
+			"each of these filepaths will be read and used as a seed for all targets.")
+	getTargets.PersistentFlags().StringVarP(&query, "query", "q", "",
+		"The query template to use when querying for changed targets")
+	getTargets.PersistentFlags().StringVarP(&Output, "output", "o", "",
+		"Filepath to write the impacted Bazel targets to, "+
+			"newline separated")
+	getTargets.PersistentFlags().StringVar(&cacheDir, "cache-dir", "",
+		"Directory to cache hashes associated with commits")
+	getTargets.PersistentFlags().BoolVar(&cacheDisabled, "nocache", false,
+		"Disables hash caching")
+}