diff --git a/DESCRIPTION b/DESCRIPTION index 9e056a6f..7e2701d9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -79,7 +79,7 @@ Suggests: RSQLite (>= 2.2.2), shiny, shinychat (>= 0.4.0), - testthat (>= 3.0.0), + testthat (>= 3.1.7), tibble, usethis Config/Needs/website: brand.yml, tidyverse/tidytemplate @@ -113,6 +113,7 @@ Collate: 'import-standalone-purrr.R' 'import-standalone-types-check.R' 'mcp.R' + 'pkg-test-reporter.R' 'task_create_btw_md.R' 'task_create_readme.R' 'task_create_skill.R' diff --git a/NEWS.md b/NEWS.md index ad25c84b..68823705 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ # btw (development version) +* Package tests now support a compact reporter that shows per-file progress, timing, warning and failure details, and final counts. `btw pkg test` uses it by default, while `btw_tool_pkg_test()` defaults to concise output for non-streaming clients; both accept other testthat reporter names (#220). + # btw 1.5.0 ## New features diff --git a/R/cli.R b/R/cli.R index 34a28c07..7ced9530 100644 --- a/R/cli.R +++ b/R/cli.R @@ -25,7 +25,7 @@ install_btw_cli <- function(destdir = NULL, ...) { "pkgload", "callr", "covr", - "testthat", + "testthat (>= 3.1.7)", "rmarkdown", "pkgsearch" )) diff --git a/R/pkg-test-reporter.R b/R/pkg-test-reporter.R new file mode 100644 index 00000000..425728a7 --- /dev/null +++ b/R/pkg-test-reporter.R @@ -0,0 +1,167 @@ +# Compact testthat reporter used by btw pkg test and btw_tool_pkg_test(). + +btw_test_name <- function(file) { + sub("\\.[rR]$", "", sub("^test-", "", basename(file))) +} + +btw_test_duration <- function(seconds) { + decimals <- if (seconds < 10) 2L else if (seconds < 100) 1L else 0L + if (round(seconds, decimals) >= 10 && decimals == 2L) { + decimals <- 1L + } + if (round(seconds, decimals) >= 100 && decimals == 1L) { + decimals <- 0L + } + sprintf(paste0("%.", decimals, "fs"), seconds) +} + +btw_test_location <- function(result) { + ref <- result$srcref + if (!inherits(ref, "srcref")) { + return("") + } + filename <- attr(ref, "srcfile")$filename + if (is.null(filename)) { + return("") + } + sprintf("%s:%s:%s", filename, ref[[1]], ref[[2]]) +} + +btw_test_type <- function(result) { + sub("^expectation_", "", class(result)[[1]]) +} + +btw_compact_reporter <- function() { + rlang::check_installed("R6") + + R6::R6Class("BtwCompactReporter", inherit = testthat::Reporter, public = list( + name = NULL, + file_id = NULL, + files = NULL, + color = NULL, + counts = NULL, + failures = NULL, + warnings = NULL, + + initialize = function() { + super$initialize() + self$capabilities$parallel_support <- TRUE + self$capabilities$parallel_updates <- TRUE + self$files <- list() + self$color <- identical(self$out, stdout()) && + sink.number() == 0L && cli::num_ansi_colors() > 1L + self$counts <- c(P = 0L, F = 0L, S = 0L, W = 0L) + self$failures <- list() + self$warnings <- list() + }, + colorize = function(text, style) { + if (!self$color) { + return(text) + } + switch( + style, + muted = cli::col_grey(text), + pass = cli::col_green(text), + fail = cli::col_red(text), + warn = cli::col_yellow(text) + ) + }, + start_file = function(name) { + self$file_id <- name + self$name <- btw_test_name(name) + # In testthat's parallel mode, start_file() is repeated for every + # event. Keep each running file's timer and counts across events. + if (!is.null(self$files[[self$file_id]])) { + return(invisible(NULL)) + } + self$files[[self$file_id]] <- list( + started = proc.time()[[3L]], + counts = c(P = 0L, F = 0L, S = 0L, W = 0L) + ) + self$cat_line(self$colorize(paste0("@ ", self$name), "muted")) + if (identical(self$out, stdout())) flush(stdout()) + }, + add_result = function(context, test, result) { + type <- btw_test_type(result) + key <- if (type %in% c("failure", "error")) { + self$failures <- c(self$failures, list(result)) + "F" + } else if (type == "skip") { + "S" + } else if (type == "warning") { + self$warnings <- c(self$warnings, list(result)) + "W" + } else { + "P" + } + file <- self$files[[self$file_id]] + file$counts[[key]] <- file$counts[[key]] + 1L + self$files[[self$file_id]] <- file + self$counts[[key]] <- self$counts[[key]] + 1L + }, + end_file = function() { + file <- self$files[[self$file_id]] + if (is.null(file)) { + return(invisible(NULL)) + } + elapsed <- proc.time()[[3L]] - file$started + counts <- file$counts + status <- if (counts[["F"]] > 0L) { + "\u2717" + } else if (counts[["W"]] > 0L) { + "!" + } else { + "\u2713" + } + counts <- counts[counts > 0L] + if (!length(counts)) { + counts <- c(P = 0L) + } + summary <- paste(vapply(names(counts), function(key) { + style <- switch(key, P = "pass", F = "fail", S = "muted", W = "warn") + self$colorize(paste0(key, ":", counts[[key]]), style) + }, ""), collapse = " ") + styled_status <- self$colorize( + status, switch(status, "\u2713" = "pass", "\u2717" = "fail", "warn") + ) + styled_time <- self$colorize(btw_test_duration(elapsed), "muted") + self$cat_line(sprintf( + "%s %s %s %s", + styled_status, self$name, styled_time, summary + )) + if (identical(self$out, stdout())) flush(stdout()) + self$files[[self$file_id]] <- NULL + }, + end_reporter = function() { + if (length(self$warnings)) { + self$cat_line() + self$cat_line(self$colorize("======== WARNINGS ========", "warn")) + for (warning in self$warnings) { + self$cat_line( + self$colorize("WARN", "warn"), ": ", + btw_test_location(warning) + ) + self$cat_line(format(warning)) + self$cat_line() + } + } + if (length(self$failures)) { + self$cat_line() + self$cat_line(self$colorize("======== FAILURES ========", "fail")) + for (failure in self$failures) { + self$cat_line( + self$colorize(toupper(btw_test_type(failure)), "fail"), + ": ", btw_test_location(failure) + ) + self$cat_line(format(failure)) + self$cat_line() + } + } + self$cat_line(sprintf( + "[ FAIL %d | WARN %d | SKIP %d | PASS %d ]", + self$counts[["F"]], self$counts[["W"]], self$counts[["S"]], self$counts[["P"]] + )) + if (identical(self$out, stdout())) flush(stdout()) + } + ))$new() +} diff --git a/R/tool-pkg-devtools.R b/R/tool-pkg-devtools.R index 71cb244a..d9fb6fef 100644 --- a/R/tool-pkg-devtools.R +++ b/R/tool-pkg-devtools.R @@ -153,12 +153,18 @@ For iterative development, use the `btw_tool_pkg_test` if available or `devtools #' Tool: Run package tests #' #' Run package tests using [devtools::test()]. Optionally filter tests by name -#' pattern. +#' pattern. The default `"minimal"` reporter returns failures and a final +#' summary without per-file progress, which suits non-streaming tool clients. +#' Use `"compact"` to include file starts, per-file results, and timings, or +#' pass a testthat reporter name. #' #' @param pkg Path to package directory. Defaults to '.'. Must be within #' current working directory. #' @param filter Optional regex to filter test files. Example: 'helper' matches #' 'test-helper.R'. +#' @param reporter Either `"minimal"` (the default), `"compact"` (per-file +#' progress and timing), or a testthat reporter name passed to +#' [devtools::test()]. #' @inheritParams btw_tool_docs_package_news #' #' @returns The output from [devtools::test()]. @@ -166,29 +172,50 @@ For iterative development, use the `btw_tool_pkg_test` if available or `devtools #' @seealso [btw_tools()] #' @family pkg tools #' @export -btw_tool_pkg_test <- function(pkg = ".", filter = NULL, `_intent`) {} +btw_tool_pkg_test <- function(pkg = ".", filter = NULL, reporter = "minimal", `_intent`) {} -btw_tool_pkg_test_impl <- function(pkg = ".", filter = NULL) { +btw_pkg_test_validate <- function(pkg, filter, reporter) { check_string(pkg) check_path_within_current_wd(pkg) + check_string(filter, allow_null = TRUE) + check_string(reporter) +} - filter_arg <- if (!is.null(filter)) { - check_string(filter) - sprintf(', filter = "%s"', filter) - } else { - "" - } +btw_pkg_test_run <- function(pkg = ".", filter = NULL, reporter = "compact") { + btw_pkg_test_validate(pkg, filter, reporter) + withr::local_envvar(TESTTHAT_PROBLEMS = "false") + + resolved_reporter <- switch( + reporter, + compact = if (utils::packageVersion("testthat") >= "3.1.7") { + btw_compact_reporter() + } else { + # Older testthat versions don't call the per-file reporter hooks. + "check" + }, + minimal = if (utils::packageVersion("testthat") >= "3.3.2") "llm" else "check", + reporter + ) + invisible(devtools::test( + pkg = pkg, + filter = filter, + stop_on_failure = FALSE, + export_all = TRUE, + reporter = resolved_reporter + )) +} - rptr <- if (utils::packageVersion("testthat") >= "3.3.2") "llm" else "check" +btw_tool_pkg_test_impl <- function(pkg = ".", filter = NULL, reporter = "minimal") { + rlang::check_installed("testthat", version = "3.1.7") + btw_pkg_test_validate(pkg, filter, reporter) + # Use one runner for both the captured tool output and the streaming CLI. code <- sprintf( - 'devtools::test(pkg = "%s"%s, stop_on_failure = FALSE, export_all = TRUE, reporter = "%s")', - pkg, - filter_arg, - rptr + "btw:::btw_pkg_test_run(pkg = %s, filter = %s, reporter = %s)", + encodeString(pkg, quote = '"'), + if (is.null(filter)) "NULL" else encodeString(filter, quote = '"'), + encodeString(reporter, quote = '"') ) - - withr::local_envvar(TESTTHAT_PROBLEMS = "false") btw_tool_run_r_impl(code) } @@ -204,7 +231,8 @@ btw_tool_pkg_test_impl <- function(pkg = ".", filter = NULL) { Runs `devtools::test()` which executes the test suite in tests/testthat/ and reports: - Number of tests passed, failed, warned, and skipped - Detailed failure messages with file locations -- Test execution time + +The `compact` reporter also shows per-file progress and execution time. The filter parameter accepts a regular expression matched against test file names after stripping the 'test-' prefix and '.R' extension. For example: - filter = 'helper' runs test-helper.R @@ -212,7 +240,7 @@ The filter parameter accepts a regular expression matched against test file name - No filter runs all tests - It is common to pair `test-{name}.R` with a source `{name}.R` file. To test this file, you can generally use filter = '{name}'. -Use `filter` when working on specific functionality to get faster feedback. The tool always runs all matching tests to completion regardless of failures.", +Use `filter` when working on specific functionality to get faster feedback. The tool always runs all matching tests to completion regardless of failures. `reporter = 'minimal'` (default) keeps output short for non-streaming clients, with failures and a final summary. Use `'compact'` for per-file progress and timing; the CLI defaults to compact. Other testthat reporter names are passed through.", annotations = ellmer::tool_annotations( title = "Testing package", read_only_hint = FALSE, @@ -227,6 +255,10 @@ Use `filter` when working on specific functionality to get faster feedback. The filter = ellmer::type_string( "Optional regex to filter test files. Example: 'helper' matches 'test-helper.R'.", required = FALSE + ), + reporter = ellmer::type_string( + "Reporter: 'minimal' (default, brief output), 'compact' (per-file progress and timing), or any testthat reporter name.", + required = FALSE ) ) ) diff --git a/exec/btw.R b/exec/btw.R index d29f9c9c..1b7c3366 100755 --- a/exec/btw.R +++ b/exec/btw.R @@ -294,12 +294,13 @@ btw_pkg_check <- function(path) { btw_output(btw:::btw_tool_pkg_check_impl(path)) } -btw_pkg_test <- function(path, filter) { - btw_output( - btw:::btw_tool_pkg_test_impl( - path, - if (has_value(filter)) filter else NULL - ) +btw_pkg_test <- function(path, filter, reporter) { + # Run directly rather than via the capturing tool, so file starts are visible + # to tail -f while tests are still running. + btw:::btw_pkg_test_run( + path, + if (has_value(filter)) filter else NULL, + reporter ) } @@ -952,11 +953,18 @@ switch( }, #| title: Run package tests + #| description: > + #| Run testthat tests. compact (default) streams file starts and results + #| with timing; minimal is a good choice for short, non-streaming output. test = { #| description: Regex to filter test files. #| short: 'f' filter <- "" - tryCatch(btw_pkg_test(path, filter), error = btw_error) + #| description: > + #| compact (default) shows per-file progress and timing. minimal shows + #| failures and a final summary. Other testthat reporter names work too. + reporter <- "compact" + tryCatch(btw_pkg_test(path, filter, reporter), error = btw_error) }, #| title: Load package with pkgload diff --git a/inst/cli-skill/r-btw-cli/SKILL.md b/inst/cli-skill/r-btw-cli/SKILL.md index 8c939e27..ac698e5c 100644 --- a/inst/cli-skill/r-btw-cli/SKILL.md +++ b/inst/cli-skill/r-btw-cli/SKILL.md @@ -30,11 +30,15 @@ Use `btw pkg` to run development tasks on an R package under active development. ``` btw pkg document [--path ] Generate roxygen2 docs btw pkg check [--path ] Run R CMD check -btw pkg test [-f ] [--path ] Run testthat tests +btw pkg test [-f ] [--reporter compact|minimal|] [--path ] Run testthat tests with live per-file progress by default btw pkg load [--path ] Load package with pkgload btw pkg coverage [--file ] [--json] Compute test coverage ``` +`btw pkg test` defaults to `--reporter compact` for live file progress and +per-file timings. Use `--reporter minimal` for a short failures-and-summary +report when you do not need to watch progress. + Use `btw pkg src` to inspect the **R namespace implementations** of installed packages (or the dev package via `.`), e.g. to understand behavior the docs don't cover. It returns exact source when available and deparsed functions diff --git a/man/btw_tool_pkg_test.Rd b/man/btw_tool_pkg_test.Rd index e65e0092..7bfc3151 100644 --- a/man/btw_tool_pkg_test.Rd +++ b/man/btw_tool_pkg_test.Rd @@ -4,7 +4,12 @@ \alias{btw_tool_pkg_test} \title{Tool: Run package tests} \usage{ -btw_tool_pkg_test(pkg = ".", filter = NULL, `_intent` = "") +btw_tool_pkg_test( + pkg = ".", + filter = NULL, + reporter = "minimal", + `_intent` = "" +) } \arguments{ \item{pkg}{Path to package directory. Defaults to '.'. Must be within @@ -13,6 +18,10 @@ current working directory.} \item{filter}{Optional regex to filter test files. Example: 'helper' matches 'test-helper.R'.} +\item{reporter}{Either \code{"minimal"} (the default), \code{"compact"} (per-file +progress and timing), or a testthat reporter name passed to +\code{\link[devtools:test]{devtools::test()}}.} + \item{_intent}{An optional string describing the intent of the tool use. When the tool is used by an LLM, the model will use this argument to explain why it called the tool.} @@ -22,7 +31,10 @@ The output from \code{\link[devtools:test]{devtools::test()}}. } \description{ Run package tests using \code{\link[devtools:test]{devtools::test()}}. Optionally filter tests by name -pattern. +pattern. The default \code{"minimal"} reporter returns failures and a final +summary without per-file progress, which suits non-streaming tool clients. +Use \code{"compact"} to include file starts, per-file results, and timings, or +pass a testthat reporter name. } \seealso{ \code{\link[=btw_tools]{btw_tools()}} diff --git a/tests/testthat/helpers.R b/tests/testthat/helpers.R index d68cdbcc..5978f1f8 100644 --- a/tests/testthat/helpers.R +++ b/tests/testthat/helpers.R @@ -62,7 +62,9 @@ scrub_system_info <- function(x) { x <- sub( sprintf( "Anthropic/%s", - ellmer::chat_anthropic(credentials = \() "not-a-real-key")$get_model() + suppressMessages( + ellmer::chat_anthropic(credentials = \() "not-a-real-key") + )$get_model() ), "Anthropic/DEFAULT_MODEL", x, diff --git a/tests/testthat/test-btw_chat_history_store.R b/tests/testthat/test-btw_chat_history_store.R index c851631a..a1d026c0 100644 --- a/tests/testthat/test-btw_chat_history_store.R +++ b/tests/testthat/test-btw_chat_history_store.R @@ -1,5 +1,7 @@ skip_if_not_installed("RSQLite") skip_if_no_shinychat_v05() +# Avoid Shiny's attachment banner when the history tests first use Shinychat. +suppressPackageStartupMessages(withr::local_package("shiny")) history_record <- function( id, diff --git a/tests/testthat/test-btw_client_app.R b/tests/testthat/test-btw_client_app.R index f15ad415..85328fc7 100644 --- a/tests/testthat/test-btw_client_app.R +++ b/tests/testthat/test-btw_client_app.R @@ -1,3 +1,8 @@ +# Avoid Shiny's attachment banner when these tests first render the app. +if (requireNamespace("shiny", quietly = TRUE)) { + suppressPackageStartupMessages(withr::local_package("shiny")) +} + test_that("app_set_disabled() namespaces controls and preserves an array payload", { message <- NULL session <- list( diff --git a/tests/testthat/test-cli.R b/tests/testthat/test-cli.R index 3d949852..5cb7b5f5 100644 --- a/tests/testthat/test-cli.R +++ b/tests/testthat/test-cli.R @@ -242,29 +242,92 @@ test_that("btw pkg check calls check impl", { expect_equal(env$path, ".") }) -test_that("btw pkg test calls test impl with filter", { - mock_filter <- NULL +test_that("btw pkg test help explains reporter trade-offs", { + result <- run_btw_subprocess("pkg", "test", "--help") + expect_equal(result$status, 0) + expect_match(result$stdout, "minimal is a good choice", fixed = TRUE) + expect_match(result$stdout, "per-file progress and timing", fixed = TRUE) + expect_match(result$stdout, '[default: "compact"]', fixed = TRUE) +}) + +test_that("btw pkg test emits file starts and completions by default", { + args <- NULL local_mocked_bindings( - btw_tool_pkg_test_impl = function(pkg, filter = NULL) { - mock_filter <<- filter - "Tests passed." + btw_pkg_test_run = function(pkg, filter = NULL, reporter = "compact") { + args <<- list(pkg = pkg, filter = filter, reporter = reporter) + cat("@ utils\n✓ utils 0.10s P:1\n") } ) env <- run_btw_quietly("pkg", "test", "-f", "utils") expect_equal(env$filter, "utils") - expect_equal(mock_filter, "utils") + expect_equal(args, list(pkg = ".", filter = "utils", reporter = "compact")) + expect_equal(env$.output, c("@ utils", "✓ utils 0.10s P:1")) }) -test_that("btw pkg test without filter passes NULL", { - mock_filter <- "SENTINEL" +test_that("btw pkg test forwards the reporter and missing filter", { + args <- NULL local_mocked_bindings( - btw_tool_pkg_test_impl = function(pkg, filter = NULL) { - mock_filter <<- filter - "Tests passed." + btw_pkg_test_run = function(pkg, filter = NULL, reporter = "compact") { + args <<- list(pkg = pkg, filter = filter, reporter = reporter) } ) - run_btw_quietly("pkg", "test") - expect_null(mock_filter) + run_btw_quietly("pkg", "test", "--reporter", "minimal") + expect_equal(args, list(pkg = ".", filter = NULL, reporter = "minimal")) +}) + +test_that("btw pkg test streams results before all files finish", { + skip_if_not_installed("processx") + pkg <- withr::local_tempdir(tmpdir = getwd()) + scripts <- file.path(pkg, "tests", "testthat") + dir.create(scripts, recursive = TRUE) + writeLines(c( + "Package: btwtestfixture", "Version: 0.0.1", "Title: Test fixture", + "Description: An isolated test package.", "License: MIT", + "Suggests: testthat", "Config/testthat/edition: 3", + "Config/testthat/parallel: true" + ), file.path(pkg, "DESCRIPTION")) + writeLines('library(testthat)\ntest_check("btwtestfixture")', file.path(pkg, "tests", "testthat.R")) + gate <- file.path(pkg, "continue") + writeLines(c( + 'test_that("slow test", {', + sprintf(' while (!file.exists(%s)) Sys.sleep(0.05)', + encodeString(gate, quote = '"')), + ' expect_true(TRUE)', + '})' + ), file.path(scripts, "test-slow.R")) + writeLines('test_that("fast test", expect_true(TRUE))', + file.path(scripts, "test-fast.R")) + + output_file <- withr::local_tempfile() + error_file <- withr::local_tempfile() + script <- sprintf( + 'pkgload::load_all(%s, quiet = TRUE); Rapp::run(%s, c("pkg", "test", "--path", %s))', + encodeString(btw_pkg_dir_resolved, quote = '"'), + encodeString(btw_cli_path(), quote = '"'), + encodeString(pkg, quote = '"') + ) + proc <- processx::process$new( + "Rscript", c("-e", script), stdout = output_file, stderr = error_file + ) + withr::defer(if (proc$is_alive()) proc$kill()) + deadline <- Sys.time() + 45 + repeat { + lines <- if (file.exists(output_file)) readLines(output_file, warn = FALSE) else character() + fast_done <- any(grepl("^✓ fast", lines)) + slow_started <- any(grepl("^@ slow", lines)) + if ((fast_done && slow_started) || !proc$is_alive() || Sys.time() > deadline) break + proc$poll_io(100) + } + expect_true(fast_done, info = paste(readLines(error_file, warn = FALSE), collapse = "\n")) + expect_true(slow_started, info = "The slow file must start before its gate is released") + expect_true(proc$is_alive(), info = "A file result should arrive while another file is running") + expect_false(any(grepl("^✓ slow", lines))) + file.create(gate) + proc$wait(timeout = 10000) + expect_false(proc$is_alive()) + expect_equal(proc$get_exit_status(), 0L, info = paste(readLines(error_file, warn = FALSE), collapse = "\n")) + expect_equal(tail(readLines(output_file, warn = FALSE), 1), + "[ FAIL 0 | WARN 0 | SKIP 0 | PASS 2 ]") }) test_that("btw pkg load calls load impl", { @@ -509,12 +572,12 @@ test_that("btw pkg src list auto-loads a dev package found in cwd", { app <- btw_cli_path() local_dev_package("devpkgone") - expect_message( + suppressMessages(expect_message( output <- capture.output( env <- Rapp::run(app, c("pkg", "src", "list", "devpkgone")) ), "Loaded in-development package" - ) + )) expect_match(paste(output, collapse = "\n"), "devpkgone_hello") }) @@ -531,7 +594,7 @@ test_that("btw pkg src get finds a dev package in an R/ subfolder", { app <- btw_cli_path() local_dev_package("devpkgthree", subdir = "R") - expect_message( + suppressMessages(expect_message( output <- capture.output( env <- Rapp::run( app, @@ -539,7 +602,7 @@ test_that("btw pkg src get finds a dev package in an R/ subfolder", { ) ), "Loaded in-development package" - ) + )) expect_match(paste(output, collapse = "\n"), "devpkgthree_hello") }) diff --git a/tests/testthat/test-pkg-test-reporter.R b/tests/testthat/test-pkg-test-reporter.R new file mode 100644 index 00000000..3b23f58f --- /dev/null +++ b/tests/testthat/test-pkg-test-reporter.R @@ -0,0 +1,92 @@ +test_that("compact reporter tracks files and prints a final summary", { + test_dir <- withr::local_tempdir() + dir.create(file.path(test_dir, "tests", "testthat"), recursive = TRUE) + scripts <- file.path(test_dir, "tests", "testthat") + writeLines(c( + 'test_that("passing", {', + ' expect_true(TRUE)', + ' skip("not applicable")', + '})' + ), file.path(scripts, "test-tool-run.R")) + writeLines(c( + 'test_that("failure and warning", {', + ' expect_true(FALSE)', + ' warning("a warning")', + '})' + ), file.path(scripts, "test-config.R")) + + output_file <- withr::local_tempfile() + withr::local_options(testthat.output_file = output_file) + reporter <- btw_compact_reporter() + suppressMessages(testthat::test_dir(scripts, reporter = reporter, stop_on_failure = FALSE)) + output <- readLines(output_file, warn = FALSE) + start <- grep("^@ ", output, value = TRUE) + done <- grep("^[✓✗!] ", output, value = TRUE) + expect_equal(start, c("@ config", "@ tool-run")) + expect_length(done, 2) + expect_match(done[[1]], "^✗ config [0-9.]+s F:1 W:1$") + expect_match(done[[2]], "^✓ tool-run [0-9.]+s P:1 S:1$") + expect_true("======== FAILURES ========" %in% output) + expect_true("======== WARNINGS ========" %in% output) + expect_equal(tail(output, 1), "[ FAIL 1 | WARN 1 | SKIP 1 | PASS 1 ]") + expect_false(any(grepl("\033", output, fixed = TRUE))) + expect_gt(which(output == "======== FAILURES ========"), + max(which(grepl("^[✓✗!] ", output)))) +}) + +test_that("files finishing out of order each get one start and completion", { + output_file <- withr::local_tempfile() + withr::local_options(testthat.output_file = output_file) + reporter <- btw_compact_reporter() + reporter$start_file("test-a.R") + reporter$start_file("test-longer.R") + reporter$start_file("test-a.R") # testthat repeats this callback in parallel mode + reporter$end_file() + reporter$start_file("test-longer.R") + reporter$end_file() + reporter$end_reporter() + + output <- readLines(output_file, warn = FALSE) + expect_equal(grep("^@ ", output, value = TRUE), c("@ a", "@ longer")) + expect_match(output[[3]], "^✓ a [0-9.]+s P:0$") + expect_match(output[[4]], "^✓ longer [0-9.]+s P:0$") + expect_equal(tail(output, 1), "[ FAIL 0 | WARN 0 | SKIP 0 | PASS 0 ]") +}) + +test_that("color styling is opt-in", { + output_file <- withr::local_tempfile() + withr::local_options(testthat.output_file = output_file) + reporter <- btw_compact_reporter() + expect_equal(reporter$colorize("0.12s", "muted"), "0.12s") + + withr::local_options(cli.num_colors = 8L) + reporter$color <- TRUE + expect_true(grepl("\033[", reporter$colorize("0.12s", "muted"), fixed = TRUE)) +}) + +test_that("compact duration uses three display digits", { + expect_equal(btw_test_duration(0.123), "0.12s") + expect_equal(btw_test_duration(1.4), "1.40s") + expect_equal(btw_test_duration(12.34), "12.3s") + expect_equal(btw_test_duration(123.4), "123s") + expect_equal(btw_test_duration(9.999), "10.0s") + expect_equal(btw_test_duration(99.999), "100s") +}) + +test_that("compact reporter displays empty files and counts errors as failures", { + test_dir <- withr::local_tempdir() + scripts <- file.path(test_dir, "tests", "testthat") + dir.create(scripts, recursive = TRUE) + writeLines("# no tests", file.path(scripts, "test-empty.R")) + writeLines('test_that("error", stop("oops"))', file.path(scripts, "test-error.R")) + + output_file <- withr::local_tempfile() + withr::local_options(testthat.output_file = output_file) + suppressMessages(testthat::test_dir( + scripts, reporter = btw_compact_reporter(), stop_on_failure = FALSE + )) + output <- readLines(output_file, warn = FALSE) + expect_true(any(grepl("^✓ empty\\s+[0-9.]+s P:0$", output))) + expect_true(any(grepl("^✗ error\\s+[0-9.]+s F:1$", output))) + expect_equal(tail(output, 1), "[ FAIL 1 | WARN 0 | SKIP 0 | PASS 0 ]") +}) diff --git a/tests/testthat/test-tool-pkg-devtools.R b/tests/testthat/test-tool-pkg-devtools.R index b0acf796..8639926c 100644 --- a/tests/testthat/test-tool-pkg-devtools.R +++ b/tests/testthat/test-tool-pkg-devtools.R @@ -152,12 +152,25 @@ test_that("btw_tool_pkg_test constructs correct code without filter", { result <- btw_tool_pkg_test_impl(".") expect_s7_class(result, BtwRunToolResult) - expect_match(result@extra$code, "devtools::test") + expect_match(result@extra$code, "btw_pkg_test_run") expect_match(result@extra$code, 'pkg = "."') - expect_match(result@extra$code, "stop_on_failure = FALSE") - expect_match(result@extra$code, "export_all = TRUE") - # Should NOT have filter argument - expect_false(grepl("filter", result@extra$code)) + expect_match(result@extra$code, "filter = NULL") + expect_match(result@extra$code, 'reporter = "minimal"') +}) + +test_that("btw_tool_pkg_test requires testthat 3.1.7", { + requirement <- NULL + local_mocked_bindings( + check_installed = function(pkg, ..., version = NULL) { + requirement <<- c(pkg = pkg, version = version) + }, + .package = "rlang" + ) + local_mocked_bindings(btw_tool_run_r_impl = function(code) code) + + btw_tool_pkg_test_impl() + + expect_equal(requirement, c(pkg = "testthat", version = "3.1.7")) }) test_that("btw_tool_pkg_test constructs correct code with filter", { @@ -177,11 +190,49 @@ test_that("btw_tool_pkg_test constructs correct code with filter", { result <- btw_tool_pkg_test_impl(".", filter = "helper") expect_s7_class(result, BtwRunToolResult) - expect_match(result@extra$code, "devtools::test") + expect_match(result@extra$code, "btw_pkg_test_run") expect_match(result@extra$code, 'pkg = "."') expect_match(result@extra$code, 'filter = "helper"') - expect_match(result@extra$code, "stop_on_failure = FALSE") - expect_match(result@extra$code, "export_all = TRUE") + expect_match(result@extra$code, 'reporter = "minimal"') +}) + +test_that("btw_tool_pkg_test forwards reporter names", { + local_mocked_bindings( + btw_tool_run_r_impl = function(code) { + BtwRunToolResult( + value = list(ContentOutput(text = "Test output")), + extra = list(code = code, status = "success", data = NULL, contents = list()) + ) + } + ) + + expect_match(btw_tool_pkg_test_impl(reporter = "minimal")@extra$code, 'reporter = "minimal"') + expect_match(btw_tool_pkg_test_impl(reporter = "compact")@extra$code, 'reporter = "compact"') + expect_match(btw_tool_pkg_test_impl(reporter = "progress")@extra$code, 'reporter = "progress"') + expect_error(btw_tool_pkg_test_impl(reporter = 1)) +}) + +test_that("package test runner resolves built-in and external reporters", { + args <- NULL + local_mocked_bindings( + test = function(...) args <<- list(...), + .package = "devtools" + ) + local_mocked_bindings( + btw_compact_reporter = function() "CUSTOM" + ) + + btw_pkg_test_run(filter = "utils") + expect_equal(args$reporter, "CUSTOM") + expect_equal(args$filter, "utils") + expect_false(args$stop_on_failure) + + btw_pkg_test_run(reporter = "minimal") + expected <- if (utils::packageVersion("testthat") >= "3.3.2") "llm" else "check" + expect_equal(args$reporter, expected) + + btw_pkg_test_run(reporter = "progress") + expect_equal(args$reporter, "progress") }) test_that("btw_tool_pkg_test handles different filter patterns", { @@ -333,12 +384,9 @@ test_that("btw_tool_pkg_test handles filter with special regex chars", { result <- btw_tool_pkg_test_impl(".", filter = "test-.*\\.R$") expect_s7_class(result, BtwRunToolResult) - # deparse() should properly quote the regex - expect_true(grepl( - 'filter = \"test-.*\\.R$\"', - result@extra$code, - fixed = TRUE - )) + # Generated R code must preserve regex backslashes when evaluated. + expect_match(result@extra$code, "filter = ", fixed = TRUE) + expect_match(result@extra$code, "test-.*\\\\.R$", fixed = TRUE) }) # Test return values -----------------------------------------------------------