Add a few utils script

scripts/bench.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Run the analyzer on a given project multiple times with different options.
+# The runs are measured using `time --verbose` and the results are stored in the
+# suggested output directory (if any) or in the current directory.
+
+usage() {
+  echo "Usage: $0 [-a dead_code_analyzer] [-o outdir] project"
+  echo ""
+  echo "Options:"
+  echo "  -a dead_code_analyzer  Select the analyzer to use (default is _build/install/default/bin/dead_code_analyzer)"
+  echo "  -o outdir  Write results files in directory outdir (default is .)"
+  echo "  -h  Display this list of options"
+}
+
+# Default values
+
+dca="_build/install/default/bin/dead_code_analyzer"
+outdir="."
+
+# Parse opt args
+
+while getopts "a:o:h" OPT
+do
+  case "$OPT" in
+    a) dca=$OPTARG ;;
+    o) outdir=$OPTARG ;;
+    h) usage; exit 0 ;;
+  esac
+done
+
+# Parse positional args
+
+shift $((OPTIND -1))
+
+if [ "$#" -ne 1 ]
+then
+  usage
+  exit 1
+fi
+
+proj="$1"
+
+# Ensure the output dir exists
+
+if [ ! -d "$outdir" ]
+then
+  mkdir -p "$outdir"
+fi
+
+# Benchamark dca on proj and store the results in outdir
+# for each flags in name_and_flags
+
+declare -A name_and_flags
+name_and_flags["default"]=""
+name_and_flags["all"]="--all"
+name_and_flags["eall"]="-a -E all"
+name_and_flags["mall"]="-a -M all"
+name_and_flags["tall"]="-a -T all"
+name_and_flags["oaall"]="-a -Oa all"
+name_and_flags["onall"]="-a -On all"
+name_and_flags["sall"]="-a -S +all"
+name_and_flags["nothing"]="--nothing"
+name_and_flags["verbose"]="-a --verbose"
+
+for name in ${!name_and_flags[@]}
+do
+  echo 'running: `' "$dca" ${name_and_flags[$name]} "$proj" '`'
+  \time --verbose "$dca" ${name_and_flags[$name]} "$proj" > "$outdir"/$name.out 2> "$outdir"/$name.err
+done

scripts/compare_bench.ml

@@ -0,0 +1,96 @@
+let usage () =
+  Printf.eprintf "Usage: ocaml %s <bench_dir1> <bench_dir2>\n" Sys.argv.(0)
+
+type errors =
+  | File_not_found of string
+  | Not_a_file of string
+
+let validate_file filepath =
+  if not (Sys.file_exists filepath) then
+    Result.error (File_not_found filepath)
+  else if not (Sys.is_regular_file filepath) then
+    Result.error (Not_a_file filepath)
+  else
+    Result.ok filepath
+
+let extract_value ~prefix line =
+  if String.starts_with ~prefix line then
+    let len_prefix = String.length prefix in
+    let value = String.sub line len_prefix (String.length line - len_prefix) in
+    Some value
+  else None
+
+let float_of_time time =
+  String.split_on_char ':' time
+  |> List.fold_left (fun acc s -> Float.fma acc 60. (float_of_string s)) 0.
+
+let float_of_mem mem = float_of_string mem
+
+let get_time_and_memory file =
+  let lines = In_channel.input_lines file in
+  let time_prefix = "\tElapsed (wall clock) time (h:mm:ss or m:ss): " in
+  let mem_prefix = "\tMaximum resident set size (kbytes): " in
+  List.fold_left (fun (time, memory) line ->
+      match time, memory with
+      | Some _, Some _ -> time, memory
+      | None, _ ->
+          let time = extract_value ~prefix:time_prefix line in
+          let time = Option.map float_of_time time in
+          time, memory
+      | Some _, None ->
+          let memory = extract_value ~prefix:mem_prefix line in
+          let memory = Option.map float_of_mem memory in
+          time, memory
+    )
+    (None, None)
+    lines
+
+let print_stats ~category metric1 metric2 =
+  let absolute_diff = metric2 -. metric1 in
+  let relative_diff = absolute_diff /. metric1 *. 100. in
+  Printf.printf "%s: %F, %F, %F, %F%%\n" category metric1 metric2 absolute_diff relative_diff
+
+let compare_files ~category bench_file1 bench_file2 =
+  match In_channel.with_open_text bench_file1 get_time_and_memory with
+  | None, _ | _, None -> ()
+  | Some time1, Some mem1 ->
+      match In_channel.with_open_text bench_file2 get_time_and_memory with
+      | None, _ | _, None -> ()
+      | Some time2, Some mem2 ->
+        print_stats ~category:(category ^ " (time)") time1 time2;
+        print_stats ~category:(category ^ " (memory)") mem1 mem2
+
+
+let compare_dirs bench_dir1 bench_dir2 =
+  let files = Sys.readdir bench_dir1 in
+  Array.iter (fun filename ->
+      if not (Filename.extension filename = ".err") then ()
+      else
+        match Filename.concat bench_dir1 filename |> validate_file with
+        | Result.Error _ -> ()
+        | Result.Ok bench_file1 ->
+            match Filename.concat bench_dir2 filename |> validate_file with
+              | Result.Error _ -> ()
+              | Result.Ok bench_file2 ->
+                  let category = Filename.remove_extension filename in
+                  compare_files ~category bench_file1 bench_file2
+    )
+    files
+
+
+let get_bench_dir arg_pos =
+  let bench_dir = Sys.argv.(arg_pos) in
+  if not (Sys.file_exists bench_dir) then
+    invalid_arg (bench_dir ^ ": directory does not exist");
+  if not (Sys.is_directory bench_dir) then
+    invalid_arg (bench_dir ^ ": is not a directory");
+  bench_dir
+
+let () =
+  if Array.length Sys.argv < 3 then (
+    usage ();
+    exit 1
+  );
+  let bench_dir1 = get_bench_dir 1 in
+  let bench_dir2 = get_bench_dir 2 in
+  compare_dirs bench_dir1 bench_dir2

scripts/reref.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Update test scenarios' .ref files to the current results
+
+make -C check stats
+
+echo "COPYING"
+cp check/threshold-3-0.5.out check/threshold-3-0.5/threshold-3-0.5.ref
+cp check/threshold-1.out check/threshold-1/threshold-1.ref
+cp check/internal.out check/internal/internal.ref
+cp check/classic.out check/classic/classic.ref
+echo "DONE"