From 494e3e4f26a39cfadcf0a9711393a1bf36f2682d Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Thu, 13 Aug 2026 16:31:28 -0500 Subject: [PATCH 01/30] refactor(imputation): Initial attempt to refactor AFT imputation modeling with conjugate gradient instead of cholesky factorization for the Newton step --- NAMESPACE | 6 + R/dataProcess.R | 50 ++- R/utils_cgsolve.R | 102 +++++ R/utils_imputation.R | 464 +++++++++++++++++++++-- inst/tinytest/test_dataProcess.R | 72 ++++ inst/tinytest/test_utils_cgsolve.R | 72 ++++ inst/tinytest/test_utils_imputation_cg.R | 127 +++++++ man/MSstatsSummarizeSingleLinear.Rd | 7 +- man/MSstatsSummarizeSingleTMP.Rd | 3 +- man/MSstatsSummarizeWithSingleCore.Rd | 3 +- man/dataProcess.Rd | 9 +- man/dot-aftGaussianDerivatives.Rd | 54 +++ man/dot-buildAFTFormula.Rd | 30 ++ man/dot-cgSolve.Rd | 47 +++ man/dot-fitSurvivalCG.Rd | 36 ++ 15 files changed, 1029 insertions(+), 53 deletions(-) create mode 100644 R/utils_cgsolve.R create mode 100644 inst/tinytest/test_utils_cgsolve.R create mode 100644 inst/tinytest/test_utils_imputation_cg.R create mode 100644 man/dot-aftGaussianDerivatives.Rd create mode 100644 man/dot-buildAFTFormula.Rd create mode 100644 man/dot-cgSolve.Rd create mode 100644 man/dot-fitSurvivalCG.Rd diff --git a/NAMESPACE b/NAMESPACE index ba6c1354..4bb4e353 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -122,14 +122,20 @@ importFrom(plotly,subplot) importFrom(preprocessCore,normalize.quantiles) importFrom(rlang,.data) importFrom(stats,dist) +importFrom(stats,dnorm) importFrom(stats,fitted) importFrom(stats,formula) importFrom(stats,hclust) importFrom(stats,lm) +importFrom(stats,lm.fit) importFrom(stats,loess) importFrom(stats,median) +importFrom(stats,model.frame) +importFrom(stats,model.matrix) +importFrom(stats,model.response) importFrom(stats,na.omit) importFrom(stats,p.adjust) +importFrom(stats,pnorm) importFrom(stats,predict) importFrom(stats,qbinom) importFrom(stats,qnorm) diff --git a/R/dataProcess.R b/R/dataProcess.R index ef23f518..e4130d5f 100755 --- a/R/dataProcess.R +++ b/R/dataProcess.R @@ -62,6 +62,11 @@ #' a logfile named `MSstats_dataProcess_log_progress.log` is created to #' track progress. Only works for Linux & Mac OS. Default is 1. #' @param aft_iterations Number of iterations for AFT model fitting. Default is 90. +#' @param aft_solver Which linear solve to use for the AFT imputation +#' model's Newton-Raphson step: "cholesky" (default) delegates to +#' \code{survival::survreg}, which solves it via Cholesky factorization. +#' "cg" solves the same Newton step with a vendored conjugate-gradient +#' routine instead - an experimental alternative, currently opt-in only. #' @inheritParams .documentFunction #' #' @importFrom utils sessionInfo @@ -130,7 +135,7 @@ dataProcess = function( equalFeatureVar = TRUE, censoredInt = "NA", MBimpute = TRUE, remove50missing = FALSE, fix_missing = NULL, maxQuantileforCensored = 0.999, use_log_file = TRUE, append = FALSE, verbose = TRUE, log_file_path = NULL, - numberOfCores = 1, aft_iterations=90 + numberOfCores = 1, aft_iterations=90, aft_solver = "cholesky" ) { MSstatsConvert::MSstatsLogsSettings(use_log_file, append, verbose, log_file_path, @@ -164,9 +169,10 @@ dataProcess = function( input = MSstatsPrepareForSummarization(input, summaryMethod, MBimpute, censoredInt, remove_uninformative_feature_outlier) summarized = tryCatch(MSstatsSummarizeWithMultipleCores(input, summaryMethod, - MBimpute, censoredInt, - remove50missing, equalFeatureVar, - numberOfCores, aft_iterations), + MBimpute, censoredInt, + remove50missing, equalFeatureVar, + numberOfCores, aft_iterations, + aft_solver), error = function(e) { print(e) NULL @@ -211,7 +217,8 @@ dataProcess = function( #' head(summarized[[1]][[1]]) # run-level summary #' MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol, - remove50missing, equal_variance, aft_iterations = 90) { + remove50missing, equal_variance, aft_iterations = 90, + aft_solver = "cholesky") { is_labeled_reference = "is_labeled_ref" %in% colnames(input) && any(input$is_labeled_ref, na.rm = TRUE) @@ -227,8 +234,8 @@ MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol for (protein_id in seq_len(num_proteins)) { single_protein = input[protein_indices[[protein_id]],] summarized_results[[protein_id]] = MSstatsSummarizeSingleTMP( - single_protein, impute, censored_symbol, remove50missing, - aft_iterations) + single_protein, impute, censored_symbol, remove50missing, + aft_iterations, aft_solver = aft_solver) setTxtProgressBar(pb, protein_id) } close(pb) @@ -237,8 +244,8 @@ MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol for (protein_id in seq_len(num_proteins)) { single_protein = input[protein_indices[[protein_id]],] summarized_result = MSstatsSummarizeSingleLinear( - single_protein, impute, censored_symbol, - remove50missing, aft_iterations) + single_protein, impute, censored_symbol, + remove50missing, aft_iterations, aft_solver = aft_solver) summarized_results[[protein_id]] = summarized_result setTxtProgressBar(pb, protein_id) @@ -256,9 +263,12 @@ MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol #' @param remove50missing if TRUE, proteins with more than 50\% missing values in each run are removed #' @param aft_iterations number of iterations for AFT model fitting #' @param equal_variances if TRUE, observation are assumed to be homoskedastic -#' +#' @param aft_solver Which linear solve to use for the AFT imputation +#' model's Newton-Raphson step: "cholesky" (default, via +#' \code{survival::survreg}) or "cg" (conjugate gradient). +#' #' @return list with protein-level data -#' +#' #' @importFrom stats xtabs #' #' @export @@ -286,7 +296,8 @@ MSstatsSummarizeSingleLinear = function(single_protein, censored_symbol, remove50missing, aft_iterations = 90, - equal_variances = TRUE) { + equal_variances = TRUE, + aft_solver = "cholesky") { ABUNDANCE = RUN = FEATURE = PROTEIN = LogIntensities = NULL cols = intersect( @@ -315,7 +326,11 @@ MSstatsSummarizeSingleLinear = function(single_protein, } else { single_protein[, cols, with = FALSE] } - survival_fit = .fitSurvival(fit_data, aft_iterations) + survival_fit = if (aft_solver == "cg") { + .fitSurvivalCG(fit_data, aft_iterations) + } else { + .fitSurvival(fit_data, aft_iterations) + } sigma2 = survival_fit$scale^2 single_protein[, c("predicted", "imputation_var") := { @@ -437,7 +452,8 @@ MSstatsSummarizeSingleLinear = function(single_protein, #' head(single_protein_summary[[1]]) #' MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, - remove50missing, aft_iterations = 90) { + remove50missing, aft_iterations = 90, + aft_solver = "cholesky") { newABUNDANCE = n_obs = n_obs_run = RUN = FEATURE = LABEL = NULL predicted = censored = NULL cols = intersect(colnames(single_protein), c("newABUNDANCE", "cen", "RUN", @@ -464,7 +480,11 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, # Try to fit survival model and catch convergence warnings survival_fit = withCallingHandlers({ - .fitSurvival(fit_data, aft_iterations) + if (aft_solver == "cg") { + .fitSurvivalCG(fit_data, aft_iterations) + } else { + .fitSurvival(fit_data, aft_iterations) + } }, warning = function(w) { if (grepl("converge", conditionMessage(w), ignore.case = TRUE)) { message("Convergence warning caught: ", conditionMessage(w)) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R new file mode 100644 index 00000000..a63c87f4 --- /dev/null +++ b/R/utils_cgsolve.R @@ -0,0 +1,102 @@ +#' Solve a symmetric positive (semi-)definite linear system via conjugate +#' gradient +#' +#' A minimal, single right-hand-side conjugate gradient solver, used as the +#' Newton-step linear solve in \code{.fitSurvivalCG}. Modeled on +#' \code{lfe::cgsolve}, stripped down to a single dense matrix and a single +#' right-hand-side vector (no multi-column batching, no \code{Matrix}-package +#' or operator/closure dispatch, no preconditioning - none of which are +#' needed for the small, dense AFT information matrices this is used on). +#' +#' @param coefficient_matrix symmetric positive (semi-)definite matrix, +#' e.g. the Hessian/information matrix from a Newton step. +#' @param right_hand_side vector the system is solved against, e.g. the +#' gradient/score vector from a Newton step. +#' @param initial_guess optional starting point for the iteration. Defaults +#' to the zero vector. +#' @param relative_tolerance how small the residual needs to shrink, +#' relative to the size of \code{right_hand_side}, before iteration stops. +#' @param max_iterations how many conjugate-gradient steps to try before +#' giving up. In exact arithmetic, conjugate gradient converges within +#' \code{nrow(coefficient_matrix)} steps, but rounding error erodes that +#' guarantee as the system grows, so the default allows for several times +#' that many steps. +#' +#' @return numeric vector solving (approximately) +#' \code{coefficient_matrix \%*\% solution = right_hand_side}. +#' +#' @keywords internal +.cgSolve = function(coefficient_matrix, right_hand_side, initial_guess = NULL, + relative_tolerance = 1e-8, + max_iterations = 10 * nrow(coefficient_matrix)) { + number_of_unknowns = nrow(coefficient_matrix) + solution = if (is.null(initial_guess)) { + rep(0, number_of_unknowns) + } else { + initial_guess + } + + # The residual measures how far the current guess is from solving the + # system. Conjugate gradient starts out searching in that direction. + residual = right_hand_side - drop(coefficient_matrix %*% solution) + search_direction = residual + residual_size = sum(residual * residual) + smallest_residual_size_seen = residual_size + + # Stop once the residual has shrunk far enough, relative to the size of + # the right-hand side (falling back to an absolute scale when that size + # is tiny). + convergence_threshold = + (relative_tolerance * max(sqrt(sum(right_hand_side^2)), 1))^2 + + for (iteration in seq_len(max_iterations)) { + if (residual_size <= convergence_threshold) { + break + } + + # How far moving along the search direction changes things, as + # measured through the matrix itself. + matrix_times_search_direction = + drop(coefficient_matrix %*% search_direction) + curvature = sum(search_direction * matrix_times_search_direction) + if (!is.finite(curvature) || curvature <= 0) { + warning(".cgSolve: coefficient_matrix is not positive definite ", + "along the current search direction; returning the ", + "best iterate found so far") + break + } + + # Move as far as possible along the search direction without + # overshooting the solution, then see how much residual remains. + step_length = residual_size / curvature + solution = solution + step_length * search_direction + residual = residual - step_length * matrix_times_search_direction + new_residual_size = sum(residual * residual) + smallest_residual_size_seen = + min(smallest_residual_size_seen, new_residual_size) + + # If the residual has grown far past its best value so far, the + # iteration is diverging (e.g. because coefficient_matrix is + # ill-conditioned) - give up and return what we have rather than + # loop until max_iterations. + if (iteration > 10 && + new_residual_size > 1e4 * smallest_residual_size_seen) { + warning(".cgSolve: residual is diverging; returning the best ", + "iterate found so far") + break + } + + # Choose the next search direction so it doesn't undo the progress + # made by earlier directions. + search_direction = residual + + (new_residual_size / residual_size) * search_direction + residual_size = new_residual_size + } + + if (residual_size > convergence_threshold) { + warning(".cgSolve: did not converge within max_iterations = ", + max_iterations, " iterations; returning the best iterate ", + "found so far") + } + solution +} diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 99bffcf1..4d378990 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -1,59 +1,455 @@ +#' Decide which predictors go into a single protein's AFT imputation model +#' +#' MSstats fits an accelerated-failure-time (AFT) model per protein to +#' impute left-censored values, and predictors are chosen based on how much +#' information is actually available: whether this is a labeled (SRM) +#' experiment with a reference channel (\code{ref_covariate}), whether +#' there is more than one feature to estimate a \code{FEATURE} effect for, +#' and whether there are enough uncensored observations to estimate that +#' effect at all. Both \code{.fitSurvival} (Cholesky-based, via +#' \code{survival::survreg}) and \code{.fitSurvivalCG} (conjugate-gradient +#' based) share this selection logic, so the two solvers always fit the +#' same model and differ only in how the Newton step is solved. +#' +#' @param input data.table with columns \code{newABUNDANCE}, \code{cen}, +#' \code{RUN}, \code{FEATURE}, \code{LABEL}, and (for labeled experiments) +#' \code{ref_covariate}. +#' +#' @return a formula whose left side is +#' \code{Surv(newABUNDANCE, cen, type = "left")}. +#' #' @importFrom data.table uniqueN -#' @importFrom survival survreg Surv +#' @importFrom survival Surv #' @keywords internal -.fitSurvival = function(input, aft_iterations) { +.buildAFTFormula = function(input) { FEATURE = RUN = NULL - + missingness_filter = is.finite(input$newABUNDANCE) n_total = nrow(input[missingness_filter, ]) n_features = data.table::uniqueN(input[missingness_filter, FEATURE]) n_runs = data.table::uniqueN(input[missingness_filter, RUN]) is_labeled = data.table::uniqueN(input$LABEL) > 1 - countdf = n_total < n_features + n_runs - 1 - # TODO: set.seed here? - set.seed(100) + # With too few uncensored observations, there isn't enough information + # left to also estimate a separate effect per feature. + not_enough_data_for_feature_effect = n_total < n_features + n_runs - 1 + if (is_labeled) { - if (length(unique(input$FEATURE)) == 1) { - # with single feature, not converge, wrong intercept - # need to check - fit = survreg(Surv(newABUNDANCE, cen, type='left') ~ RUN + ref_covariate, - data = input, dist = "gaussian", - control = list(maxiter=aft_iterations)) + if (length(unique(input$FEATURE)) == 1 || + not_enough_data_for_feature_effect) { + # with a single feature (or too little data), a FEATURE term + # either adds nothing or keeps the model from converging / + # gives it the wrong intercept - need to check + Surv(newABUNDANCE, cen, type = "left") ~ RUN + ref_covariate } else { - if (countdf) { - fit = survreg(Surv(newABUNDANCE, cen, type='left') ~ RUN + ref_covariate, - data = input, dist = "gaussian", - control = list(maxiter=aft_iterations)) - } else { - fit = survreg(Surv(newABUNDANCE, cen, type='left') ~ FEATURE + RUN + ref_covariate, - data = input, dist = "gaussian", - control = list(maxiter=aft_iterations)) - } + Surv(newABUNDANCE, cen, type = "left") ~ + FEATURE + RUN + ref_covariate } } else { - if (n_features == 1L) { - fit = survreg(Surv(newABUNDANCE, cen, type = "left") ~ RUN, - data = input, dist = "gaussian", - control = list(maxiter=aft_iterations)) + if (n_features == 1L || not_enough_data_for_feature_effect) { + Surv(newABUNDANCE, cen, type = "left") ~ RUN } else { - if (countdf) { - fit = survreg(Surv(newABUNDANCE, cen, type = "left") ~ RUN, - data = input, dist = "gaussian", - control = list(maxiter=aft_iterations)) - } else { - fit = survreg(Surv(newABUNDANCE, cen, type = "left") ~ FEATURE + RUN, - data = input, dist = "gaussian", - control = list(maxiter=aft_iterations)) - } + Surv(newABUNDANCE, cen, type = "left") ~ FEATURE + RUN } } +} + +#' @importFrom survival survreg +#' @keywords internal +.fitSurvival = function(input, aft_iterations) { + # TODO: set.seed here? + set.seed(100) + fit = survreg(.buildAFTFormula(input), data = input, dist = "gaussian", + control = list(maxiter = aft_iterations)) fit$y = NULL fit$linear.predictors = NULL fit } +#' Per-observation log-likelihood and derivatives for a Gaussian AFT model +#' +#' Computes what a Newton-Raphson step needs at the current parameter +#' guess: the log-likelihood, its first derivative with respect to the +#' linear predictor and to the log of the scale parameter, and the +#' corresponding second derivatives - all summed/assembled later into the +#' score vector and information matrix by \code{.fitSurvivalCG}. This only +#' covers the two cases MSstats' AFT imputation actually uses: an exact +#' (uncensored) observation, or one left-censored below a detection-limit +#' ceiling (\code{Surv(y, cen, type = "left")} with \code{cen == 0}). +#' +#' The formulas are transcribed term-for-term from \code{survival}'s own +#' C implementation (\code{survregc1.c}'s \code{gauss_d} function and its +#' "exact"/"left censored" cases) rather than re-derived by hand, since a +#' hand re-derivation is an easy place to introduce a sign error; this +#' function's correctness is instead checked against numerical +#' differentiation of the log-likelihood (see +#' \code{test_utils_imputation_cg.R}). +#' +#' @param linear_predictor current linear predictor +#' (\code{model_matrix \%*\% coefficients}). +#' @param log_scale current log of the scale parameter. +#' @param observed_value observed value (or, for censored rows, the +#' detection-limit ceiling substituted in by +#' \code{.setCensoredByThreshold}). +#' @param exact_indicator \code{1} for an exact/uncensored observation, +#' \code{0} for one left-censored below \code{observed_value}. +#' +#' @return a list with the total \code{log_likelihood}, and +#' per-observation vectors \code{gradient_wrt_linear_predictor}, +#' \code{second_derivative_wrt_linear_predictor}, +#' \code{gradient_wrt_log_scale}, \code{second_derivative_wrt_log_scale}, +#' and \code{cross_derivative} +#' (d2 log_likelihood / d linear_predictor d log_scale). +#' +#' @importFrom stats dnorm pnorm +#' @keywords internal +.aftGaussianDerivatives = function(linear_predictor, log_scale, + observed_value, exact_indicator) { + scale = exp(log_scale) + inverse_scale_squared = 1 / scale^2 + + # How far the observation sits from its predicted value, in raw units + # and in standard deviations. + distance_from_prediction = observed_value - linear_predictor + standardized_distance = distance_from_prediction / scale + + density_at_standardized_distance = dnorm(standardized_distance) + cumulative_probability_at_standardized_distance = + pnorm(standardized_distance) + is_exact_observation = (exact_indicator == 1) + + # --- exact (uncensored) observations -------------------------------- + # log-likelihood contribution is log(density) - log(scale); what + # follows is that expression's derivatives wrt linear_predictor and + # log_scale. + exact_log_likelihood = + log(density_at_standardized_distance) - log_scale + exact_gradient_wrt_linear_predictor = standardized_distance / scale + exact_log_density_curvature = + (standardized_distance^2 - 1) * inverse_scale_squared + exact_second_derivative_wrt_linear_predictor = + exact_log_density_curvature - + exact_gradient_wrt_linear_predictor^2 + exact_gradient_wrt_log_scale_before_adjustment = + exact_gradient_wrt_linear_predictor * distance_from_prediction + exact_cross_derivative = + distance_from_prediction * exact_log_density_curvature - + exact_gradient_wrt_linear_predictor * + (exact_gradient_wrt_log_scale_before_adjustment + 1) + exact_second_derivative_wrt_log_scale = + distance_from_prediction^2 * exact_log_density_curvature - + exact_gradient_wrt_log_scale_before_adjustment * + (1 + exact_gradient_wrt_log_scale_before_adjustment) + exact_gradient_wrt_log_scale = + exact_gradient_wrt_log_scale_before_adjustment - 1 + + # Guard against the density underflowing to exactly zero (only + # happens for astronomically large |standardized_distance|, e.g. from + # a wild early Newton guess). Any reasonable derivative works here, + # since the collapsed log-likelihood itself is what triggers + # step-halving. + exact_density_underflowed = density_at_standardized_distance <= 0 + exact_log_likelihood = + ifelse(exact_density_underflowed, -200, exact_log_likelihood) + exact_gradient_wrt_linear_predictor = ifelse( + exact_density_underflowed, -standardized_distance / scale, + exact_gradient_wrt_linear_predictor) + exact_second_derivative_wrt_linear_predictor = ifelse( + exact_density_underflowed, -1 / scale, + exact_second_derivative_wrt_linear_predictor) + exact_gradient_wrt_log_scale = + ifelse(exact_density_underflowed, 0, exact_gradient_wrt_log_scale) + exact_cross_derivative = + ifelse(exact_density_underflowed, 0, exact_cross_derivative) + exact_second_derivative_wrt_log_scale = ifelse( + exact_density_underflowed, 0, + exact_second_derivative_wrt_log_scale) + + # --- left-censored observations (true value <= the recorded ceiling) - + # log-likelihood contribution is log(Phi(standardized_distance)); + # "censoring_hazard" plays the same role for these rows that the + # density itself plays above. + censored_log_likelihood = + log(cumulative_probability_at_standardized_distance) + censoring_hazard = density_at_standardized_distance / + (cumulative_probability_at_standardized_distance * scale) + censored_gradient_wrt_linear_predictor = -censoring_hazard + censored_log_density_curvature = + -standardized_distance * density_at_standardized_distance * + inverse_scale_squared / + cumulative_probability_at_standardized_distance + censored_second_derivative_wrt_linear_predictor = + censored_log_density_curvature - + censored_gradient_wrt_linear_predictor^2 + censored_gradient_wrt_log_scale = + censored_gradient_wrt_linear_predictor * distance_from_prediction + censored_cross_derivative = + distance_from_prediction * censored_log_density_curvature - + censored_gradient_wrt_linear_predictor * + (censored_gradient_wrt_log_scale + 1) + censored_second_derivative_wrt_log_scale = + distance_from_prediction^2 * censored_log_density_curvature - + censored_gradient_wrt_log_scale * (1 + censored_gradient_wrt_log_scale) + + # Same underflow guard as above, triggered when the cumulative + # probability collapses to zero (standardized_distance very + # negative). + censored_probability_underflowed = + cumulative_probability_at_standardized_distance <= 0 + censored_log_likelihood = ifelse( + censored_probability_underflowed, -200, censored_log_likelihood) + censored_gradient_wrt_linear_predictor = ifelse( + censored_probability_underflowed, -standardized_distance / scale, + censored_gradient_wrt_linear_predictor) + censored_second_derivative_wrt_linear_predictor = ifelse( + censored_probability_underflowed, 0, + censored_second_derivative_wrt_linear_predictor) + censored_gradient_wrt_log_scale = ifelse( + censored_probability_underflowed, 0, censored_gradient_wrt_log_scale) + censored_cross_derivative = ifelse( + censored_probability_underflowed, 0, censored_cross_derivative) + censored_second_derivative_wrt_log_scale = ifelse( + censored_probability_underflowed, 0, + censored_second_derivative_wrt_log_scale) + + list( + log_likelihood = sum(ifelse( + is_exact_observation, exact_log_likelihood, + censored_log_likelihood)), + gradient_wrt_linear_predictor = ifelse( + is_exact_observation, exact_gradient_wrt_linear_predictor, + censored_gradient_wrt_linear_predictor), + second_derivative_wrt_linear_predictor = ifelse( + is_exact_observation, + exact_second_derivative_wrt_linear_predictor, + censored_second_derivative_wrt_linear_predictor), + gradient_wrt_log_scale = ifelse( + is_exact_observation, exact_gradient_wrt_log_scale, + censored_gradient_wrt_log_scale), + second_derivative_wrt_log_scale = ifelse( + is_exact_observation, exact_second_derivative_wrt_log_scale, + censored_second_derivative_wrt_log_scale), + cross_derivative = ifelse( + is_exact_observation, exact_cross_derivative, + censored_cross_derivative) + ) +} + +#' Fit a Gaussian, left-censored AFT model with a conjugate-gradient +#' Newton step +#' +#' An alternative to \code{.fitSurvival} for exactly the same imputation +#' model (Gaussian accelerated-failure-time regression, left-censoring +#' only, chosen by the same \code{.buildAFTFormula} both solvers share), +#' used when \code{aft_solver = "cg"}. It runs the same kind of +#' Newton-Raphson iteration \code{survival::survreg} does - repeatedly +#' solving \code{information_matrix \%*\% step = gradient} for the next +#' set of coefficients - but performs that linear solve with the +#' conjugate-gradient routine \code{.cgSolve} instead of the Cholesky +#' factorization \code{survreg} uses internally. The returned object is +#' classed \code{"survreg"} and carries the fields \code{predict.survreg} +#' needs, so it is a drop-in replacement anywhere \code{.fitSurvival}'s +#' result is used. +#' +#' @param input data.table, the same shape \code{.fitSurvival} expects. +#' @param aft_iterations maximum number of Newton-Raphson iterations. +#' @param convergence_tolerance stop once the change in log-likelihood +#' between iterations falls below this (matches the default +#' \code{rel.tolerance} in \code{survival::survreg.control}). +#' +#' @return a fitted model of class \code{"survreg"}. +#' +#' @importFrom stats model.frame model.matrix model.response lm.fit sd +#' @keywords internal +.fitSurvivalCG = function(input, aft_iterations, + convergence_tolerance = 1e-9) { + model_frame = model.frame(.buildAFTFormula(input), data = input) + model_terms = attr(model_frame, "terms") + design_matrix = model.matrix(model_terms, model_frame) + number_of_coefficients = ncol(design_matrix) + + response = model.response(model_frame) + observed_value = response[, 1] + exact_indicator = response[, 2] + + # Initial guess: an ordinary least-squares fit for the regression + # coefficients (treating the detection-limit ceiling already + # substituted into censored rows as if it were observed), and the + # residual standard deviation for the scale parameter. A + # rank-deficient design leaves some coefficients unidentified + # (reported as NA by lm.fit); start those at zero. + initial_fit = lm.fit(design_matrix, observed_value) + coefficients = initial_fit$coefficients + coefficients[!is.finite(coefficients)] = 0 + residual_standard_deviation = sd(initial_fit$residuals) + log_scale = log(max(residual_standard_deviation, 1e-4)) + + evaluate_log_likelihood_and_derivatives = function(coefficients, + log_scale) { + .aftGaussianDerivatives( + drop(design_matrix %*% coefficients), log_scale, + observed_value, exact_indicator) + } + + build_gradient = function(derivatives) { + c(as.vector(crossprod( + design_matrix, derivatives$gradient_wrt_linear_predictor)), + sum(derivatives$gradient_wrt_log_scale)) + } + + build_information_matrix = function(derivatives) { + # Regression block: -t(X) %*% diag(second_derivative) %*% X, + # computed without forming the diagonal matrix explicitly. + regression_block = -crossprod( + design_matrix, + design_matrix * derivatives$second_derivative_wrt_linear_predictor) + cross_block = -as.vector( + crossprod(design_matrix, derivatives$cross_derivative)) + scale_block = -sum(derivatives$second_derivative_wrt_log_scale) + rbind(cbind(regression_block, cross_block), + c(cross_block, scale_block)) + } + + is_finite_fit = function(derivatives) { + is.finite(derivatives$log_likelihood) && + all(is.finite(derivatives$gradient_wrt_linear_predictor)) && + all(is.finite(derivatives$gradient_wrt_log_scale)) && + all(is.finite(derivatives$second_derivative_wrt_linear_predictor)) && + all(is.finite(derivatives$second_derivative_wrt_log_scale)) + } + + # A Newton step away from the optimum, the exact information matrix + # is not guaranteed to be positive definite. survival::survreg falls + # back, in that situation, to the sum of the outer products of each + # observation's own contribution to the gradient - always positive + # semi-definite by construction, and equal to the exact information + # matrix in expectation (this is the classic Gauss-Newton / BHHH + # approximation). Mirror that fallback here. + build_gauss_newton_approximation = function(derivatives) { + per_observation_gradient_contributions = cbind( + design_matrix * derivatives$gradient_wrt_linear_predictor, + derivatives$gradient_wrt_log_scale) + crossprod(per_observation_gradient_contributions) + } + + solve_newton_step = function(information_matrix, derivatives, gradient) { + information_matrix_is_not_positive_definite = FALSE + step = withCallingHandlers( + .cgSolve(information_matrix, gradient), + warning = function(w) { + if (grepl("not positive definite", conditionMessage(w))) { + information_matrix_is_not_positive_definite <<- TRUE + } + invokeRestart("muffleWarning") + }) + if (information_matrix_is_not_positive_definite) { + step = .cgSolve(build_gauss_newton_approximation(derivatives), + gradient) + } + step + } + + current_fit = + evaluate_log_likelihood_and_derivatives(coefficients, log_scale) + current_log_likelihood = current_fit$log_likelihood + number_of_iterations_used = 0 + converged = FALSE + + for (iteration in seq_len(aft_iterations)) { + number_of_iterations_used = iteration + gradient = build_gradient(current_fit) + information_matrix = build_information_matrix(current_fit) + newton_step = + solve_newton_step(information_matrix, current_fit, gradient) + + candidate_coefficients = + coefficients + newton_step[seq_len(number_of_coefficients)] + candidate_log_scale = + log_scale + newton_step[number_of_coefficients + 1] + + # Step-halving: if the Newton step overshoots (a non-finite or + # decreasing log-likelihood), back the trial point off toward the + # last accepted one, mirroring survival::survreg's own recovery + # strategy (survreg6.c) rather than simply rejecting the step + # outright. + number_of_halvings = 0 + halving_exhausted = FALSE + repeat { + candidate_fit = evaluate_log_likelihood_and_derivatives( + candidate_coefficients, candidate_log_scale) + candidate_improves = is_finite_fit(candidate_fit) && + candidate_fit$log_likelihood >= current_log_likelihood + if (candidate_improves) { + break + } + number_of_halvings = number_of_halvings + 1 + if (number_of_halvings > 30) { + halving_exhausted = TRUE + break + } + if (number_of_halvings == 1 && + (log_scale - candidate_log_scale) > 1.1) { + # a single huge drop in scale is the most common cause of + # a bad trial; keep the first back-off from cutting scale + # by more than a factor of exp(1.1), same as survreg6.c + candidate_log_scale = log_scale - 1.1 + } + candidate_coefficients = + (candidate_coefficients + 2 * coefficients) / 3 + candidate_log_scale = (candidate_log_scale + 2 * log_scale) / 3 + } + + if (halving_exhausted) { + break + } + + relative_change = + abs(1 - current_log_likelihood / candidate_fit$log_likelihood) + absolute_change = + abs(candidate_fit$log_likelihood - current_log_likelihood) + + coefficients = candidate_coefficients + log_scale = candidate_log_scale + current_fit = candidate_fit + current_log_likelihood = candidate_fit$log_likelihood + + if (relative_change <= convergence_tolerance || + absolute_change <= convergence_tolerance) { + converged = TRUE + break + } + } + + if (!converged) { + warning("AFT model (CG solver) ran out of iterations and did not ", + "converge") + } + + final_information_matrix = build_information_matrix(current_fit) + variance_covariance_matrix = tryCatch( + solve(final_information_matrix), + error = function(e) MASS::ginv(final_information_matrix)) + + fitted_coefficients = coefficients + names(fitted_coefficients) = colnames(design_matrix) + + is_factor_column = vapply(model_frame, is.factor, logical(1)) + + fit = list( + coefficients = fitted_coefficients, + var = variance_covariance_matrix, + scale = exp(log_scale), + terms = model_terms, + xlevels = lapply(model_frame[is_factor_column], levels), + dist = "gaussian", + iter = number_of_iterations_used, + loglik = current_log_likelihood + ) + class(fit) = "survreg" + fit +} + #' Get predicted values from a survival model #' @param input data.table #' @return numeric vector of predictions diff --git a/inst/tinytest/test_dataProcess.R b/inst/tinytest/test_dataProcess.R index e0113862..60df17f7 100644 --- a/inst/tinytest/test_dataProcess.R +++ b/inst/tinytest/test_dataProcess.R @@ -423,3 +423,75 @@ expect_true( length(l_cens_pred) > 0 && all(is.finite(l_cens_pred)), info = "MSstatsSummarizeSingleTMP SRM: censored L rows must receive a finite imputed predicted value" ) + +# --- Same SRM imputation, but via aft_solver = "cg" ------------------------ +# Same invariants must hold (H never imputed, L gets a finite prediction), +# and the imputed values themselves should closely match the default +# aft_solver = "cholesky" path, since both solve the same Newton step. +# +# make_srm_impute_input()'s uncensored values are an exactly noise-free +# linear function of RUN, which makes the Gaussian scale MLE degenerate +# (unbounded as residuals -> 0). That's fine for the qualitative H/L +# invariant checks above, but not a meaningful numeric comparison between +# solvers, so a little jitter is added here to make the fit well-posed. + +make_srm_impute_input_with_noise <- function(seed) { + input <- make_srm_impute_input() + set.seed(seed) + input[cen == 1L, + newABUNDANCE := newABUNDANCE + rnorm(.N, sd = 0.01)] + input +} + +result_srm_imp_chol_noisy <- MSstatsSummarizeSingleTMP( + make_srm_impute_input_with_noise(seed = 1), + impute = TRUE, + censored_symbol = "NA", + remove50missing = FALSE, + aft_iterations = 90, + aft_solver = "cholesky" +) +result_srm_imp_cg <- MSstatsSummarizeSingleTMP( + make_srm_impute_input_with_noise(seed = 1), + impute = TRUE, + censored_symbol = "NA", + remove50missing = FALSE, + aft_iterations = 90, + aft_solver = "cg" +) + +survival_srm_chol_noisy <- result_srm_imp_chol_noisy[[2]] +survival_srm_cg <- result_srm_imp_cg[[2]] + +h_cens_pred_cg <- survival_srm_cg[ + as.character(FEATURE) == "F1" & + as.character(LABEL) == "H" & + as.character(RUN) == "R1", + predicted +] +expect_true( + length(h_cens_pred_cg) > 0 && all(is.na(h_cens_pred_cg)), + info = "MSstatsSummarizeSingleTMP SRM (aft_solver = cg): censored H rows must NOT receive an imputed predicted value" +) + +l_cens_pred_cg <- survival_srm_cg[ + as.character(FEATURE) == "F1" & + as.character(LABEL) == "L" & + as.character(RUN) == "R2", + predicted +] +l_cens_pred_chol_noisy <- survival_srm_chol_noisy[ + as.character(FEATURE) == "F1" & + as.character(LABEL) == "L" & + as.character(RUN) == "R2", + predicted +] +expect_true( + length(l_cens_pred_cg) > 0 && all(is.finite(l_cens_pred_cg)), + info = "MSstatsSummarizeSingleTMP SRM (aft_solver = cg): censored L rows must receive a finite imputed predicted value" +) +expect_equal( + l_cens_pred_cg, l_cens_pred_chol_noisy, tolerance = 1e-4, + check.attributes = FALSE, + info = "MSstatsSummarizeSingleTMP SRM: aft_solver = cg should closely match aft_solver = cholesky" +) diff --git a/inst/tinytest/test_utils_cgsolve.R b/inst/tinytest/test_utils_cgsolve.R new file mode 100644 index 00000000..89bc2a65 --- /dev/null +++ b/inst/tinytest/test_utils_cgsolve.R @@ -0,0 +1,72 @@ +# Tests for .cgSolve(), the vendored conjugate-gradient linear solver used +# as the Newton-step solve in .fitSurvivalCG(). + +make_random_spd_matrix <- function(size, seed, ridge = 0.01) { + set.seed(seed) + random_factor <- matrix(rnorm(size * size), size, size) + random_factor %*% t(random_factor) + diag(size) * ridge +} + +for (size in c(2, 5, 10, 30, 80)) { + coefficient_matrix <- make_random_spd_matrix(size, seed = size) + set.seed(size + 1000) + right_hand_side <- rnorm(size) + + cg_solution <- MSstats:::.cgSolve(coefficient_matrix, right_hand_side) + exact_solution <- solve(coefficient_matrix, right_hand_side) + + expect_equal( + cg_solution, exact_solution, tolerance = 1e-6, + info = paste0(".cgSolve should match solve() on a random SPD ", + "system of size ", size) + ) +} + +# --- near-singular system: still returns a finite result, with a warning --- + +near_singular_matrix <- make_random_spd_matrix(10, seed = 42) +near_singular_matrix[1, ] <- 0 +near_singular_matrix[, 1] <- 0 +set.seed(43) +right_hand_side <- rnorm(10) + +expect_warning( + solution <- MSstats:::.cgSolve(near_singular_matrix, right_hand_side), + info = paste("A singular coefficient_matrix should warn rather than", + "error or hang") +) +expect_true( + all(is.finite(solution)), + info = "A singular system should still return a finite (partial) solution" +) + +# --- an initial guess that is already the solution converges immediately --- + +exact_matrix <- make_random_spd_matrix(6, seed = 7) +set.seed(8) +exact_rhs <- rnorm(6) +exact_answer <- solve(exact_matrix, exact_rhs) + +solution_from_exact_start <- MSstats:::.cgSolve( + exact_matrix, exact_rhs, initial_guess = exact_answer) +expect_equal( + solution_from_exact_start, exact_answer, tolerance = 1e-8, + info = "Starting from the exact solution should return it unchanged" +) + +# --- relative_tolerance controls how tightly the system is solved --- + +loose_matrix <- make_random_spd_matrix(20, seed = 99) +set.seed(100) +loose_rhs <- rnorm(20) +exact_loose_answer <- solve(loose_matrix, loose_rhs) + +loose_solution <- MSstats:::.cgSolve(loose_matrix, loose_rhs, relative_tolerance = 1e-2) +tight_solution <- MSstats:::.cgSolve(loose_matrix, loose_rhs, relative_tolerance = 1e-10) + +loose_error <- max(abs(loose_solution - exact_loose_answer)) +tight_error <- max(abs(tight_solution - exact_loose_answer)) +expect_true( + tight_error < loose_error, + info = "A tighter relative_tolerance should produce a more accurate solution" +) diff --git a/inst/tinytest/test_utils_imputation_cg.R b/inst/tinytest/test_utils_imputation_cg.R new file mode 100644 index 00000000..4f57bfc0 --- /dev/null +++ b/inst/tinytest/test_utils_imputation_cg.R @@ -0,0 +1,127 @@ +# Tests that .fitSurvivalCG() - the conjugate-gradient alternative to +# .fitSurvival() - fits the same model, and agrees numerically with it. +# +# The scenarios below reuse the noiseless fixtures from +# test_utils_imputation.R purely to check that .fitSurvivalCG() selects the +# same predictors as .fitSurvival() (via the shared .buildAFTFormula()). +# For numeric agreement on the fitted values themselves, a Gaussian AFT +# model needs actual residual variation to estimate - a noiseless design +# has a degenerate (unbounded) scale MLE, so a second set of fixtures below +# adds realistic noise and left-censoring before comparing coefficients, +# scale, and predictions. + +make_surv_labeled_single <- function() { + runs <- paste0("R", 1:3) + dt <- data.table::rbindlist(list( + data.table::data.table( + FEATURE = factor(rep("F1", 9)), + RUN = factor(rep(runs, each = 3)), + LABEL = "H", + newABUNDANCE = seq(10.1, by = 0.1, length.out = 9), + cen = 1L + ), + data.table::data.table( + FEATURE = factor(rep("F1", 9)), + RUN = factor(rep(runs, each = 3)), + LABEL = "L", + newABUNDANCE = seq(14.1, by = 0.1, length.out = 9), + cen = 1L + ) + )) + ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") + dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) + dt +} + +make_surv_unlabeled_multi_welldetermined <- function() { + dt <- data.table::CJ( + FEATURE = paste0("F", 1:3), + RUN = paste0("R", 1:5) + ) + dt[, FEATURE := factor(FEATURE)] + dt[, RUN := factor(RUN)] + dt[, LABEL := "L"] + dt[, newABUNDANCE := seq(10, by = 0.5, length.out = .N)] + dt[, cen := 1L] + dt +} + +# --- .fitSurvivalCG() selects the same predictors as .fitSurvival() ------- + +coef_names <- function(fit) names(coef(fit)) + +for (make_input in list(make_surv_labeled_single, + make_surv_unlabeled_multi_welldetermined)) { + input <- make_input() + chol_names <- sort(coef_names(MSstats:::.fitSurvival(input, 90))) + cg_names <- sort(coef_names(MSstats:::.fitSurvivalCG(input, 90))) + expect_equal( + cg_names, chol_names, + info = ".fitSurvivalCG must select the same predictors as .fitSurvival" + ) +} + +# --- numeric agreement on realistic (noisy, censored) data ---------------- + +make_noisy_censored_input <- function(seed, is_labeled) { + set.seed(seed) + features <- paste0("F", 1:3) + runs <- paste0("R", 1:4) + labels <- if (is_labeled) c("H", "L") else "L" + dt <- data.table::CJ(FEATURE = features, RUN = runs, LABEL = labels) + dt[, FEATURE := factor(FEATURE)] + dt[, RUN := factor(RUN)] + dt[, newABUNDANCE := + 10 + as.integer(FEATURE) + as.integer(RUN) * 0.5 + + ifelse(LABEL == "L", 4, 0) + rnorm(.N, sd = 0.7)] + dt[, cen := 1L] + censoring_threshold <- stats::quantile(dt$newABUNDANCE, 0.2) + dt[newABUNDANCE < censoring_threshold, cen := 0L] + dt[cen == 0L, newABUNDANCE := censoring_threshold] + if (is_labeled) { + ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") + dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) + } + dt +} + +check_solvers_agree <- function(input, tolerance, label) { + fit_cholesky <- MSstats:::.fitSurvival(input, 90) + fit_cg <- MSstats:::.fitSurvivalCG(input, 90) + + matched_names <- names(fit_cholesky$coefficients) + expect_equal( + fit_cg$coefficients[matched_names], + fit_cholesky$coefficients, + tolerance = tolerance, check.attributes = FALSE, + info = paste(label, ": coefficients should match .fitSurvival") + ) + expect_equal( + fit_cg$scale, fit_cholesky$scale, tolerance = tolerance, + check.attributes = FALSE, + info = paste(label, ": scale should match .fitSurvival") + ) + + predictions_cholesky <- predict(fit_cholesky, newdata = input, se.fit = TRUE) + predictions_cg <- predict(fit_cg, newdata = input, se.fit = TRUE) + expect_equal( + predictions_cg$fit, predictions_cholesky$fit, + tolerance = tolerance, check.attributes = FALSE, + info = paste(label, ": predicted values should match .fitSurvival") + ) + expect_equal( + predictions_cg$se.fit, predictions_cholesky$se.fit, + tolerance = tolerance, check.attributes = FALSE, + info = paste(label, ": prediction standard errors should match", + ".fitSurvival") + ) +} + +check_solvers_agree( + make_noisy_censored_input(seed = 1, is_labeled = TRUE), + tolerance = 1e-4, label = "labeled, noisy, censored" +) +check_solvers_agree( + make_noisy_censored_input(seed = 2, is_labeled = FALSE), + tolerance = 1e-4, label = "unlabeled, noisy, censored" +) diff --git a/man/MSstatsSummarizeSingleLinear.Rd b/man/MSstatsSummarizeSingleLinear.Rd index 8595bc16..0f0056dd 100644 --- a/man/MSstatsSummarizeSingleLinear.Rd +++ b/man/MSstatsSummarizeSingleLinear.Rd @@ -10,7 +10,8 @@ MSstatsSummarizeSingleLinear( censored_symbol, remove50missing, aft_iterations = 90, - equal_variances = TRUE + equal_variances = TRUE, + aft_solver = "cholesky" ) } \arguments{ @@ -25,6 +26,10 @@ MSstatsSummarizeSingleLinear( \item{aft_iterations}{number of iterations for AFT model fitting} \item{equal_variances}{if TRUE, observation are assumed to be homoskedastic} + +\item{aft_solver}{Which linear solve to use for the AFT imputation +model's Newton-Raphson step: "cholesky" (default, via +\code{survival::survreg}) or "cg" (conjugate gradient).} } \value{ list with protein-level data diff --git a/man/MSstatsSummarizeSingleTMP.Rd b/man/MSstatsSummarizeSingleTMP.Rd index cd115723..55038a02 100644 --- a/man/MSstatsSummarizeSingleTMP.Rd +++ b/man/MSstatsSummarizeSingleTMP.Rd @@ -9,7 +9,8 @@ MSstatsSummarizeSingleTMP( impute, censored_symbol, remove50missing, - aft_iterations = 90 + aft_iterations = 90, + aft_solver = "cholesky" ) } \arguments{ diff --git a/man/MSstatsSummarizeWithSingleCore.Rd b/man/MSstatsSummarizeWithSingleCore.Rd index fc711d42..278ae8e1 100644 --- a/man/MSstatsSummarizeWithSingleCore.Rd +++ b/man/MSstatsSummarizeWithSingleCore.Rd @@ -11,7 +11,8 @@ MSstatsSummarizeWithSingleCore( censored_symbol, remove50missing, equal_variance, - aft_iterations = 90 + aft_iterations = 90, + aft_solver = "cholesky" ) } \arguments{ diff --git a/man/dataProcess.Rd b/man/dataProcess.Rd index 1ed4fe03..5f306a4e 100644 --- a/man/dataProcess.Rd +++ b/man/dataProcess.Rd @@ -25,7 +25,8 @@ dataProcess( verbose = TRUE, log_file_path = NULL, numberOfCores = 1, - aft_iterations = 90 + aft_iterations = 90, + aft_solver = "cholesky" ) } \arguments{ @@ -121,6 +122,12 @@ a logfile named `MSstats_dataProcess_log_progress.log` is created to track progress. Only works for Linux & Mac OS. Default is 1.} \item{aft_iterations}{Number of iterations for AFT model fitting. Default is 90.} + +\item{aft_solver}{Which linear solve to use for the AFT imputation +model's Newton-Raphson step: "cholesky" (default) delegates to +\code{survival::survreg}, which solves it via Cholesky factorization. +"cg" solves the same Newton step with a vendored conjugate-gradient +routine instead - an experimental alternative, currently opt-in only.} } \value{ A list containing: diff --git a/man/dot-aftGaussianDerivatives.Rd b/man/dot-aftGaussianDerivatives.Rd new file mode 100644 index 00000000..768e1dee --- /dev/null +++ b/man/dot-aftGaussianDerivatives.Rd @@ -0,0 +1,54 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils_imputation.R +\name{.aftGaussianDerivatives} +\alias{.aftGaussianDerivatives} +\title{Per-observation log-likelihood and derivatives for a Gaussian AFT model} +\usage{ +.aftGaussianDerivatives( + linear_predictor, + log_scale, + observed_value, + exact_indicator +) +} +\arguments{ +\item{linear_predictor}{current linear predictor +(\code{model_matrix \%*\% coefficients}).} + +\item{log_scale}{current log of the scale parameter.} + +\item{observed_value}{observed value (or, for censored rows, the +detection-limit ceiling substituted in by +\code{.setCensoredByThreshold}).} + +\item{exact_indicator}{\code{1} for an exact/uncensored observation, +\code{0} for one left-censored below \code{observed_value}.} +} +\value{ +a list with the total \code{log_likelihood}, and +per-observation vectors \code{gradient_wrt_linear_predictor}, +\code{second_derivative_wrt_linear_predictor}, +\code{gradient_wrt_log_scale}, \code{second_derivative_wrt_log_scale}, +and \code{cross_derivative} +(d2 log_likelihood / d linear_predictor d log_scale). +} +\description{ +Computes what a Newton-Raphson step needs at the current parameter +guess: the log-likelihood, its first derivative with respect to the +linear predictor and to the log of the scale parameter, and the +corresponding second derivatives - all summed/assembled later into the +score vector and information matrix by \code{.fitSurvivalCG}. This only +covers the two cases MSstats' AFT imputation actually uses: an exact +(uncensored) observation, or one left-censored below a detection-limit +ceiling (\code{Surv(y, cen, type = "left")} with \code{cen == 0}). +} +\details{ +The formulas are transcribed term-for-term from \code{survival}'s own +C implementation (\code{survregc1.c}'s \code{gauss_d} function and its +"exact"/"left censored" cases) rather than re-derived by hand, since a +hand re-derivation is an easy place to introduce a sign error; this +function's correctness is instead checked against numerical +differentiation of the log-likelihood (see +\code{test_utils_imputation_cg.R}). +} +\keyword{internal} diff --git a/man/dot-buildAFTFormula.Rd b/man/dot-buildAFTFormula.Rd new file mode 100644 index 00000000..d387353a --- /dev/null +++ b/man/dot-buildAFTFormula.Rd @@ -0,0 +1,30 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils_imputation.R +\name{.buildAFTFormula} +\alias{.buildAFTFormula} +\title{Decide which predictors go into a single protein's AFT imputation model} +\usage{ +.buildAFTFormula(input) +} +\arguments{ +\item{input}{data.table with columns \code{newABUNDANCE}, \code{cen}, +\code{RUN}, \code{FEATURE}, \code{LABEL}, and (for labeled experiments) +\code{ref_covariate}.} +} +\value{ +a formula whose left side is +\code{Surv(newABUNDANCE, cen, type = "left")}. +} +\description{ +MSstats fits an accelerated-failure-time (AFT) model per protein to +impute left-censored values, and predictors are chosen based on how much +information is actually available: whether this is a labeled (SRM) +experiment with a reference channel (\code{ref_covariate}), whether +there is more than one feature to estimate a \code{FEATURE} effect for, +and whether there are enough uncensored observations to estimate that +effect at all. Both \code{.fitSurvival} (Cholesky-based, via +\code{survival::survreg}) and \code{.fitSurvivalCG} (conjugate-gradient +based) share this selection logic, so the two solvers always fit the +same model and differ only in how the Newton step is solved. +} +\keyword{internal} diff --git a/man/dot-cgSolve.Rd b/man/dot-cgSolve.Rd new file mode 100644 index 00000000..5d846263 --- /dev/null +++ b/man/dot-cgSolve.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils_cgsolve.R +\name{.cgSolve} +\alias{.cgSolve} +\title{Solve a symmetric positive (semi-)definite linear system via conjugate +gradient} +\usage{ +.cgSolve( + coefficient_matrix, + right_hand_side, + initial_guess = NULL, + relative_tolerance = 1e-08, + max_iterations = 10 * nrow(coefficient_matrix) +) +} +\arguments{ +\item{coefficient_matrix}{symmetric positive (semi-)definite matrix, +e.g. the Hessian/information matrix from a Newton step.} + +\item{right_hand_side}{vector the system is solved against, e.g. the +gradient/score vector from a Newton step.} + +\item{initial_guess}{optional starting point for the iteration. Defaults +to the zero vector.} + +\item{relative_tolerance}{how small the residual needs to shrink, +relative to the size of \code{right_hand_side}, before iteration stops.} + +\item{max_iterations}{how many conjugate-gradient steps to try before +giving up. In exact arithmetic, conjugate gradient converges within +\code{nrow(coefficient_matrix)} steps, but rounding error erodes that +guarantee as the system grows, so the default allows for several times +that many steps.} +} +\value{ +numeric vector solving (approximately) +\code{coefficient_matrix \%*\% solution = right_hand_side}. +} +\description{ +A minimal, single right-hand-side conjugate gradient solver, used as the +Newton-step linear solve in \code{.fitSurvivalCG}. Modeled on +\code{lfe::cgsolve}, stripped down to a single dense matrix and a single +right-hand-side vector (no multi-column batching, no \code{Matrix}-package +or operator/closure dispatch, no preconditioning - none of which are +needed for the small, dense AFT information matrices this is used on). +} +\keyword{internal} diff --git a/man/dot-fitSurvivalCG.Rd b/man/dot-fitSurvivalCG.Rd new file mode 100644 index 00000000..98ebeb0a --- /dev/null +++ b/man/dot-fitSurvivalCG.Rd @@ -0,0 +1,36 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils_imputation.R +\name{.fitSurvivalCG} +\alias{.fitSurvivalCG} +\title{Fit a Gaussian, left-censored AFT model with a conjugate-gradient +Newton step} +\usage{ +.fitSurvivalCG(input, aft_iterations, convergence_tolerance = 1e-09) +} +\arguments{ +\item{input}{data.table, the same shape \code{.fitSurvival} expects.} + +\item{aft_iterations}{maximum number of Newton-Raphson iterations.} + +\item{convergence_tolerance}{stop once the change in log-likelihood +between iterations falls below this (matches the default +\code{rel.tolerance} in \code{survival::survreg.control}).} +} +\value{ +a fitted model of class \code{"survreg"}. +} +\description{ +An alternative to \code{.fitSurvival} for exactly the same imputation +model (Gaussian accelerated-failure-time regression, left-censoring +only, chosen by the same \code{.buildAFTFormula} both solvers share), +used when \code{aft_solver = "cg"}. It runs the same kind of +Newton-Raphson iteration \code{survival::survreg} does - repeatedly +solving \code{information_matrix \%*\% step = gradient} for the next +set of coefficients - but performs that linear solve with the +conjugate-gradient routine \code{.cgSolve} instead of the Cholesky +factorization \code{survreg} uses internally. The returned object is +classed \code{"survreg"} and carries the fields \code{predict.survreg} +needs, so it is a drop-in replacement anywhere \code{.fitSurvival}'s +result is used. +} +\keyword{internal} From b54f2566150a77c84288dd649c91949a50b72cee Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Thu, 13 Aug 2026 17:51:03 -0500 Subject: [PATCH 02/30] add adjustments w.r.t. logging and pcg option --- R/dataProcess.R | 48 +++++--- R/utils_cgsolve.R | 70 +++++++++-- R/utils_imputation.R | 143 ++++++++++++++++++++--- inst/tinytest/test_utils_cgsolve.R | 90 ++++++++++++-- inst/tinytest/test_utils_imputation_cg.R | 68 ++++++++++- man/MSstatsSummarizeSingleLinear.Rd | 10 +- man/MSstatsSummarizeSingleTMP.Rd | 3 +- man/MSstatsSummarizeWithSingleCore.Rd | 3 +- man/dataProcess.Rd | 14 ++- man/dot-cgSolve.Rd | 31 ++++- man/dot-fitAFTModel.Rd | 36 ++++++ man/dot-fitSurvivalCG.Rd | 29 ++++- 12 files changed, 472 insertions(+), 73 deletions(-) create mode 100644 man/dot-fitAFTModel.Rd diff --git a/R/dataProcess.R b/R/dataProcess.R index e4130d5f..2a97689a 100755 --- a/R/dataProcess.R +++ b/R/dataProcess.R @@ -66,7 +66,15 @@ #' model's Newton-Raphson step: "cholesky" (default) delegates to #' \code{survival::survreg}, which solves it via Cholesky factorization. #' "cg" solves the same Newton step with a vendored conjugate-gradient -#' routine instead - an experimental alternative, currently opt-in only. +#' routine instead; "pcg" is the same conjugate-gradient routine with a +#' Jacobi (inverse-diagonal) preconditioner, which can reduce the number +#' of conjugate-gradient iterations needed. "cg"/"pcg" are experimental +#' alternatives, currently opt-in only. +#' @param aft_verbose If \code{TRUE} and \code{aft_solver} is "cg" or +#' "pcg", \code{message()} per-Newton-iteration conjugate-gradient +#' iteration counts and timing for every protein fit - useful for +#' evaluating solver time complexity, but produces one block of output +#' per protein, so leave at the default \code{FALSE} for routine runs. #' @inheritParams .documentFunction #' #' @importFrom utils sessionInfo @@ -135,7 +143,8 @@ dataProcess = function( equalFeatureVar = TRUE, censoredInt = "NA", MBimpute = TRUE, remove50missing = FALSE, fix_missing = NULL, maxQuantileforCensored = 0.999, use_log_file = TRUE, append = FALSE, verbose = TRUE, log_file_path = NULL, - numberOfCores = 1, aft_iterations=90, aft_solver = "cholesky" + numberOfCores = 1, aft_iterations=90, aft_solver = "cholesky", + aft_verbose = FALSE ) { MSstatsConvert::MSstatsLogsSettings(use_log_file, append, verbose, log_file_path, @@ -172,7 +181,7 @@ dataProcess = function( MBimpute, censoredInt, remove50missing, equalFeatureVar, numberOfCores, aft_iterations, - aft_solver), + aft_solver, aft_verbose), error = function(e) { print(e) NULL @@ -218,7 +227,7 @@ dataProcess = function( #' MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol, remove50missing, equal_variance, aft_iterations = 90, - aft_solver = "cholesky") { + aft_solver = "cholesky", aft_verbose = FALSE) { is_labeled_reference = "is_labeled_ref" %in% colnames(input) && any(input$is_labeled_ref, na.rm = TRUE) @@ -235,7 +244,8 @@ MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol single_protein = input[protein_indices[[protein_id]],] summarized_results[[protein_id]] = MSstatsSummarizeSingleTMP( single_protein, impute, censored_symbol, remove50missing, - aft_iterations, aft_solver = aft_solver) + aft_iterations, aft_solver = aft_solver, + aft_verbose = aft_verbose) setTxtProgressBar(pb, protein_id) } close(pb) @@ -245,7 +255,8 @@ MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol single_protein = input[protein_indices[[protein_id]],] summarized_result = MSstatsSummarizeSingleLinear( single_protein, impute, censored_symbol, - remove50missing, aft_iterations, aft_solver = aft_solver) + remove50missing, aft_iterations, aft_solver = aft_solver, + aft_verbose = aft_verbose) summarized_results[[protein_id]] = summarized_result setTxtProgressBar(pb, protein_id) @@ -265,7 +276,11 @@ MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol #' @param equal_variances if TRUE, observation are assumed to be homoskedastic #' @param aft_solver Which linear solve to use for the AFT imputation #' model's Newton-Raphson step: "cholesky" (default, via -#' \code{survival::survreg}) or "cg" (conjugate gradient). +#' \code{survival::survreg}), "cg" (conjugate gradient), or "pcg" +#' (conjugate gradient with a Jacobi/inverse-diagonal preconditioner). +#' @param aft_verbose If \code{TRUE} and \code{aft_solver} is "cg" or +#' "pcg", log per-Newton-iteration conjugate-gradient diagnostics for +#' every protein fit. See \code{.fitSurvivalCG}'s \code{verbose}. #' #' @return list with protein-level data #' @@ -297,7 +312,8 @@ MSstatsSummarizeSingleLinear = function(single_protein, remove50missing, aft_iterations = 90, equal_variances = TRUE, - aft_solver = "cholesky") { + aft_solver = "cholesky", + aft_verbose = FALSE) { ABUNDANCE = RUN = FEATURE = PROTEIN = LogIntensities = NULL cols = intersect( @@ -326,11 +342,8 @@ MSstatsSummarizeSingleLinear = function(single_protein, } else { single_protein[, cols, with = FALSE] } - survival_fit = if (aft_solver == "cg") { - .fitSurvivalCG(fit_data, aft_iterations) - } else { - .fitSurvival(fit_data, aft_iterations) - } + survival_fit = .fitAFTModel(fit_data, aft_iterations, aft_solver, + aft_verbose) sigma2 = survival_fit$scale^2 single_protein[, c("predicted", "imputation_var") := { @@ -453,7 +466,8 @@ MSstatsSummarizeSingleLinear = function(single_protein, #' MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, remove50missing, aft_iterations = 90, - aft_solver = "cholesky") { + aft_solver = "cholesky", + aft_verbose = FALSE) { newABUNDANCE = n_obs = n_obs_run = RUN = FEATURE = LABEL = NULL predicted = censored = NULL cols = intersect(colnames(single_protein), c("newABUNDANCE", "cen", "RUN", @@ -480,11 +494,7 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, # Try to fit survival model and catch convergence warnings survival_fit = withCallingHandlers({ - if (aft_solver == "cg") { - .fitSurvivalCG(fit_data, aft_iterations) - } else { - .fitSurvival(fit_data, aft_iterations) - } + .fitAFTModel(fit_data, aft_iterations, aft_solver, aft_verbose) }, warning = function(w) { if (grepl("converge", conditionMessage(w), ignore.case = TRUE)) { message("Convergence warning caught: ", conditionMessage(w)) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index a63c87f4..4384245c 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -5,8 +5,10 @@ #' Newton-step linear solve in \code{.fitSurvivalCG}. Modeled on #' \code{lfe::cgsolve}, stripped down to a single dense matrix and a single #' right-hand-side vector (no multi-column batching, no \code{Matrix}-package -#' or operator/closure dispatch, no preconditioning - none of which are -#' needed for the small, dense AFT information matrices this is used on). +#' or operator/closure dispatch - neither is needed for the small, dense AFT +#' information matrices this is used on). Optionally applies a Jacobi +#' (inverse-diagonal) preconditioner, which \code{lfe::cgsolve} does not +#' support at all. #' #' @param coefficient_matrix symmetric positive (semi-)definite matrix, #' e.g. the Hessian/information matrix from a Newton step. @@ -16,19 +18,35 @@ #' to the zero vector. #' @param relative_tolerance how small the residual needs to shrink, #' relative to the size of \code{right_hand_side}, before iteration stops. +#' Always judged on the true (unpreconditioned) residual, so this means the +#' same thing whether or not \code{use_jacobi_preconditioner} is set. #' @param max_iterations how many conjugate-gradient steps to try before #' giving up. In exact arithmetic, conjugate gradient converges within #' \code{nrow(coefficient_matrix)} steps, but rounding error erodes that #' guarantee as the system grows, so the default allows for several times #' that many steps. +#' @param use_jacobi_preconditioner if \code{TRUE}, precondition with the +#' inverse of \code{coefficient_matrix}'s own diagonal - cheap to apply, +#' and often enough to cut down the number of iterations needed when the +#' diagonal dominates (as it typically does for an AFT information matrix, +#' where each parameter's own curvature tends to be much larger than its +#' cross-terms with the other parameters). Defaults to \code{FALSE}, which +#' reduces exactly to plain (unpreconditioned) conjugate gradient. #' -#' @return numeric vector solving (approximately) -#' \code{coefficient_matrix \%*\% solution = right_hand_side}. +#' @return a list with: \code{solution}, the numeric vector solving +#' (approximately) \code{coefficient_matrix \%*\% solution = +#' right_hand_side}; \code{iterations}, how many conjugate-gradient steps +#' were actually taken; \code{converged}, whether the residual tolerance +#' was met; and \code{positive_definite}, whether \code{coefficient_matrix} +#' behaved as positive definite throughout (a caller can fall back to a +#' different matrix, e.g. a Gauss-Newton approximation, when this is +#' \code{FALSE}). #' #' @keywords internal .cgSolve = function(coefficient_matrix, right_hand_side, initial_guess = NULL, relative_tolerance = 1e-8, - max_iterations = 10 * nrow(coefficient_matrix)) { + max_iterations = 10 * nrow(coefficient_matrix), + use_jacobi_preconditioner = FALSE) { number_of_unknowns = nrow(coefficient_matrix) solution = if (is.null(initial_guess)) { rep(0, number_of_unknowns) @@ -36,11 +54,25 @@ initial_guess } + apply_preconditioner = if (use_jacobi_preconditioner) { + diagonal = diag(coefficient_matrix) + inverse_diagonal = ifelse( + is.finite(diagonal) & diagonal > 0, 1 / diagonal, 1) + function(vector) inverse_diagonal * vector + } else { + identity + } + # The residual measures how far the current guess is from solving the - # system. Conjugate gradient starts out searching in that direction. + # system. Conjugate gradient starts out searching in the + # preconditioner-adjusted residual direction (with no preconditioner, + # this is just the residual itself). residual = right_hand_side - drop(coefficient_matrix %*% solution) - search_direction = residual + preconditioned_residual = apply_preconditioner(residual) + search_direction = preconditioned_residual residual_size = sum(residual * residual) + residual_dot_preconditioned_residual = + sum(residual * preconditioned_residual) smallest_residual_size_seen = residual_size # Stop once the residual has shrunk far enough, relative to the size of @@ -49,10 +81,14 @@ convergence_threshold = (relative_tolerance * max(sqrt(sum(right_hand_side^2)), 1))^2 + positive_definite = TRUE + iterations_used = 0 + for (iteration in seq_len(max_iterations)) { if (residual_size <= convergence_threshold) { break } + iterations_used = iteration # How far moving along the search direction changes things, as # measured through the matrix itself. @@ -60,6 +96,7 @@ drop(coefficient_matrix %*% search_direction) curvature = sum(search_direction * matrix_times_search_direction) if (!is.finite(curvature) || curvature <= 0) { + positive_definite = FALSE warning(".cgSolve: coefficient_matrix is not positive definite ", "along the current search direction; returning the ", "best iterate found so far") @@ -68,7 +105,7 @@ # Move as far as possible along the search direction without # overshooting the solution, then see how much residual remains. - step_length = residual_size / curvature + step_length = residual_dot_preconditioned_residual / curvature solution = solution + step_length * search_direction residual = residual - step_length * matrix_times_search_direction new_residual_size = sum(residual * residual) @@ -88,15 +125,24 @@ # Choose the next search direction so it doesn't undo the progress # made by earlier directions. - search_direction = residual + - (new_residual_size / residual_size) * search_direction + new_preconditioned_residual = apply_preconditioner(residual) + new_residual_dot_preconditioned_residual = + sum(residual * new_preconditioned_residual) + search_direction = new_preconditioned_residual + + (new_residual_dot_preconditioned_residual / + residual_dot_preconditioned_residual) * search_direction residual_size = new_residual_size + residual_dot_preconditioned_residual = + new_residual_dot_preconditioned_residual } - if (residual_size > convergence_threshold) { + converged = residual_size <= convergence_threshold + if (!converged && positive_definite) { warning(".cgSolve: did not converge within max_iterations = ", max_iterations, " iterations; returning the best iterate ", "found so far") } - solution + + list(solution = solution, iterations = iterations_used, + converged = converged, positive_definite = positive_definite) } diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 4d378990..9aceda7d 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -257,17 +257,45 @@ #' @param convergence_tolerance stop once the change in log-likelihood #' between iterations falls below this (matches the default #' \code{rel.tolerance} in \code{survival::survreg.control}). +#' @param use_jacobi_preconditioner if \code{TRUE}, precondition every +#' conjugate-gradient solve with the inverse of the current information +#' matrix's own diagonal (see \code{.cgSolve}'s +#' \code{use_jacobi_preconditioner}). This is what \code{aft_solver = +#' "pcg"} enables, versus plain conjugate gradient for \code{"cg"}. +#' @param verbose if \code{TRUE}, \code{message()} a line per +#' Newton-Raphson iteration - conjugate-gradient iterations used, whether +#' the Gauss-Newton fallback (see below) was needed, elapsed time, and the +#' resulting log-likelihood - plus a one-line summary once fitting +#' finishes. Meant for evaluating how solver choice and problem size +#' trade off against iteration count and wall time, not for routine use +#' (this fits one protein at a time, so it is easy to generate a line per +#' protein across a whole \code{dataProcess()} run). #' -#' @return a fitted model of class \code{"survreg"}. +#' @return a fitted model of class \code{"survreg"}, with one added field: +#' \code{cg_diagnostics}, a data.frame with one row per Newton-Raphson +#' iteration recording the conjugate-gradient iteration counts and timing +#' described above (populated regardless of \code{verbose}, so it can be +#' inspected/aggregated programmatically after the fact). #' #' @importFrom stats model.frame model.matrix model.response lm.fit sd #' @keywords internal .fitSurvivalCG = function(input, aft_iterations, - convergence_tolerance = 1e-9) { + convergence_tolerance = 1e-9, + use_jacobi_preconditioner = FALSE, + verbose = FALSE) { model_frame = model.frame(.buildAFTFormula(input), data = input) model_terms = attr(model_frame, "terms") design_matrix = model.matrix(model_terms, model_frame) number_of_coefficients = ncol(design_matrix) + number_of_parameters = number_of_coefficients + 1 + number_of_observations = nrow(design_matrix) + + if (verbose) { + message(sprintf( + "[AFT-CG] starting fit: %d observations, %d parameters, preconditioner = %s", + number_of_observations, number_of_parameters, + if (use_jacobi_preconditioner) "jacobi" else "none")) + } response = model.response(model_frame) observed_value = response[, 1] @@ -333,21 +361,43 @@ crossprod(per_observation_gradient_contributions) } - solve_newton_step = function(information_matrix, derivatives, gradient) { - information_matrix_is_not_positive_definite = FALSE - step = withCallingHandlers( - .cgSolve(information_matrix, gradient), + # A "not positive definite" result is expected, handled control flow + # here (the Gauss-Newton fallback below exists for exactly that case), + # so its warning is muffled; a genuine "did not converge within + # max_iterations" is not expected/handled, so that warning still + # propagates normally. + cg_solve_muffling_pd_warning = function(...) { + withCallingHandlers( + .cgSolve(...), warning = function(w) { if (grepl("not positive definite", conditionMessage(w))) { - information_matrix_is_not_positive_definite <<- TRUE + invokeRestart("muffleWarning") } - invokeRestart("muffleWarning") }) - if (information_matrix_is_not_positive_definite) { - step = .cgSolve(build_gauss_newton_approximation(derivatives), - gradient) + } + + # Returns the Newton step, plus how much conjugate-gradient work it + # took to get there - primary_iterations/fallback_iterations and + # used_fallback are the numbers verbose logging (below) reports, so a + # caller can see how solver choice and problem size trade off against + # iteration count. + solve_newton_step = function(information_matrix, derivatives, gradient) { + primary_solve = cg_solve_muffling_pd_warning( + information_matrix, gradient, + use_jacobi_preconditioner = use_jacobi_preconditioner) + if (primary_solve$positive_definite) { + list(step = primary_solve$solution, + primary_iterations = primary_solve$iterations, + used_fallback = FALSE, fallback_iterations = 0L) + } else { + fallback_solve = .cgSolve( + build_gauss_newton_approximation(derivatives), gradient, + use_jacobi_preconditioner = use_jacobi_preconditioner) + list(step = fallback_solve$solution, + primary_iterations = primary_solve$iterations, + used_fallback = TRUE, + fallback_iterations = fallback_solve$iterations) } - step } current_fit = @@ -355,18 +405,38 @@ current_log_likelihood = current_fit$log_likelihood number_of_iterations_used = 0 converged = FALSE + cg_diagnostics = vector("list", aft_iterations) for (iteration in seq_len(aft_iterations)) { number_of_iterations_used = iteration + iteration_start_time = Sys.time() + gradient = build_gradient(current_fit) information_matrix = build_information_matrix(current_fit) newton_step = solve_newton_step(information_matrix, current_fit, gradient) + elapsed_seconds = + as.numeric(Sys.time() - iteration_start_time, units = "secs") + cg_diagnostics[[iteration]] = data.frame( + newton_iteration = iteration, + cg_iterations = newton_step$primary_iterations + + newton_step$fallback_iterations, + used_gauss_newton_fallback = newton_step$used_fallback, + elapsed_seconds = elapsed_seconds) + if (verbose) { + message(sprintf( + "[AFT-CG] newton iter %d: cg iterations = %d%s, %.4f sec", + iteration, + newton_step$primary_iterations + newton_step$fallback_iterations, + if (newton_step$used_fallback) " (Gauss-Newton fallback used)" else "", + elapsed_seconds)) + } + candidate_coefficients = - coefficients + newton_step[seq_len(number_of_coefficients)] + coefficients + newton_step$step[seq_len(number_of_coefficients)] candidate_log_scale = - log_scale + newton_step[number_of_coefficients + 1] + log_scale + newton_step$step[number_of_coefficients + 1] # Step-halving: if the Newton step overshoots (a non-finite or # decreasing log-likelihood), back the trial point off toward the @@ -426,6 +496,17 @@ "converge") } + cg_diagnostics = do.call( + rbind, cg_diagnostics[seq_len(number_of_iterations_used)]) + + if (verbose) { + message(sprintf( + paste0("[AFT-CG] finished: %d newton iterations, ", + "%d total cg iterations, %.4f sec total, converged = %s"), + number_of_iterations_used, sum(cg_diagnostics$cg_iterations), + sum(cg_diagnostics$elapsed_seconds), converged)) + } + final_information_matrix = build_information_matrix(current_fit) variance_covariance_matrix = tryCatch( solve(final_information_matrix), @@ -444,12 +525,44 @@ xlevels = lapply(model_frame[is_factor_column], levels), dist = "gaussian", iter = number_of_iterations_used, - loglik = current_log_likelihood + loglik = current_log_likelihood, + cg_diagnostics = cg_diagnostics ) class(fit) = "survreg" fit } +#' Fit the AFT imputation model with the requested solver +#' +#' Shared dispatch used by both \code{MSstatsSummarizeSingleLinear} and +#' \code{MSstatsSummarizeSingleTMP} so the \code{aft_solver}/ +#' \code{aft_verbose} logic lives in one place instead of being duplicated +#' at both call sites. +#' +#' @param input data.table, the same shape \code{.fitSurvival} expects. +#' @param aft_iterations maximum number of iterations for AFT model fitting. +#' @param aft_solver "cholesky" (default, via \code{survival::survreg}), +#' "cg" (conjugate gradient), or "pcg" (conjugate gradient with a +#' Jacobi/inverse-diagonal preconditioner). +#' @param aft_verbose passed through to \code{.fitSurvivalCG}'s +#' \code{verbose} when \code{aft_solver} is "cg" or "pcg"; has no effect +#' for "cholesky". +#' +#' @return a fitted model of class \code{"survreg"}. +#' +#' @keywords internal +.fitAFTModel = function(input, aft_iterations, aft_solver = "cholesky", + aft_verbose = FALSE) { + if (aft_solver == "pcg") { + .fitSurvivalCG(input, aft_iterations, + use_jacobi_preconditioner = TRUE, verbose = aft_verbose) + } else if (aft_solver == "cg") { + .fitSurvivalCG(input, aft_iterations, verbose = aft_verbose) + } else { + .fitSurvival(input, aft_iterations) + } +} + #' Get predicted values from a survival model #' @param input data.table #' @return numeric vector of predictions diff --git a/inst/tinytest/test_utils_cgsolve.R b/inst/tinytest/test_utils_cgsolve.R index 89bc2a65..902c7c4b 100644 --- a/inst/tinytest/test_utils_cgsolve.R +++ b/inst/tinytest/test_utils_cgsolve.R @@ -1,5 +1,6 @@ # Tests for .cgSolve(), the vendored conjugate-gradient linear solver used -# as the Newton-step solve in .fitSurvivalCG(). +# as the Newton-step solve in .fitSurvivalCG(). Returns a list: +# solution/iterations/converged/positive_definite. make_random_spd_matrix <- function(size, seed, ridge = 0.01) { set.seed(seed) @@ -12,14 +13,23 @@ for (size in c(2, 5, 10, 30, 80)) { set.seed(size + 1000) right_hand_side <- rnorm(size) - cg_solution <- MSstats:::.cgSolve(coefficient_matrix, right_hand_side) + cg_result <- MSstats:::.cgSolve(coefficient_matrix, right_hand_side) exact_solution <- solve(coefficient_matrix, right_hand_side) expect_equal( - cg_solution, exact_solution, tolerance = 1e-6, + cg_result$solution, exact_solution, tolerance = 1e-6, info = paste0(".cgSolve should match solve() on a random SPD ", "system of size ", size) ) + expect_true( + cg_result$converged && cg_result$positive_definite, + info = paste0("A well-conditioned SPD system of size ", size, + " should report converged/positive_definite = TRUE") + ) + expect_true( + cg_result$iterations >= 1 && cg_result$iterations <= size * 10, + info = "iterations should be a small positive count, not the default cap" + ) } # --- near-singular system: still returns a finite result, with a warning --- @@ -31,14 +41,19 @@ set.seed(43) right_hand_side <- rnorm(10) expect_warning( - solution <- MSstats:::.cgSolve(near_singular_matrix, right_hand_side), + singular_result <- MSstats:::.cgSolve(near_singular_matrix, right_hand_side), info = paste("A singular coefficient_matrix should warn rather than", "error or hang") ) expect_true( - all(is.finite(solution)), + all(is.finite(singular_result$solution)), info = "A singular system should still return a finite (partial) solution" ) +expect_false( + singular_result$converged && singular_result$positive_definite, + info = paste("A singular coefficient_matrix should signal trouble via", + "converged = FALSE and/or positive_definite = FALSE") +) # --- an initial guess that is already the solution converges immediately --- @@ -47,12 +62,16 @@ set.seed(8) exact_rhs <- rnorm(6) exact_answer <- solve(exact_matrix, exact_rhs) -solution_from_exact_start <- MSstats:::.cgSolve( +result_from_exact_start <- MSstats:::.cgSolve( exact_matrix, exact_rhs, initial_guess = exact_answer) expect_equal( - solution_from_exact_start, exact_answer, tolerance = 1e-8, + result_from_exact_start$solution, exact_answer, tolerance = 1e-8, info = "Starting from the exact solution should return it unchanged" ) +expect_equal( + result_from_exact_start$iterations, 0, + info = "Starting from the exact solution should take zero iterations" +) # --- relative_tolerance controls how tightly the system is solved --- @@ -61,12 +80,61 @@ set.seed(100) loose_rhs <- rnorm(20) exact_loose_answer <- solve(loose_matrix, loose_rhs) -loose_solution <- MSstats:::.cgSolve(loose_matrix, loose_rhs, relative_tolerance = 1e-2) -tight_solution <- MSstats:::.cgSolve(loose_matrix, loose_rhs, relative_tolerance = 1e-10) +loose_result <- MSstats:::.cgSolve( + loose_matrix, loose_rhs, relative_tolerance = 1e-2) +tight_result <- MSstats:::.cgSolve( + loose_matrix, loose_rhs, relative_tolerance = 1e-10) -loose_error <- max(abs(loose_solution - exact_loose_answer)) -tight_error <- max(abs(tight_solution - exact_loose_answer)) +loose_error <- max(abs(loose_result$solution - exact_loose_answer)) +tight_error <- max(abs(tight_result$solution - exact_loose_answer)) expect_true( tight_error < loose_error, info = "A tighter relative_tolerance should produce a more accurate solution" ) + +# --- Jacobi preconditioner: same answer, fewer or equal iterations -------- +# on a diagonally-dominant system (where a diagonal preconditioner is most +# effective), preconditioned CG should converge in no more iterations than +# plain CG, and to the same solution. + +make_diagonally_dominant_matrix <- function(size, seed) { + set.seed(seed) + matrix_off_diagonal <- matrix(runif(size * size, -0.1, 0.1), size, size) + matrix_off_diagonal <- (matrix_off_diagonal + t(matrix_off_diagonal)) / 2 + diag(matrix_off_diagonal) <- 0 + diag(size) * runif(size, 5, 10) + matrix_off_diagonal +} + +dominant_matrix <- make_diagonally_dominant_matrix(40, seed = 11) +set.seed(12) +dominant_rhs <- rnorm(40) +exact_dominant_answer <- solve(dominant_matrix, dominant_rhs) + +plain_cg_result <- MSstats:::.cgSolve(dominant_matrix, dominant_rhs) +preconditioned_result <- MSstats:::.cgSolve( + dominant_matrix, dominant_rhs, use_jacobi_preconditioner = TRUE) + +expect_equal( + preconditioned_result$solution, exact_dominant_answer, tolerance = 1e-6, + info = "Preconditioned CG should still match solve() on a diagonally dominant system" +) +expect_true( + preconditioned_result$iterations <= plain_cg_result$iterations, + info = paste("Jacobi preconditioning should not need more iterations", + "than plain CG on a diagonally dominant system (plain =", + plain_cg_result$iterations, ", preconditioned =", + preconditioned_result$iterations, ")") +) + +# A degenerate (all-zero) diagonal entry should not blow up the +# preconditioner (falls back to an identity-like scale of 1 for that entry). +degenerate_diagonal_matrix <- make_random_spd_matrix(8, seed = 55) +degenerate_diagonal_matrix[3, 3] <- 0 +set.seed(56) +degenerate_rhs <- rnorm(8) +expect_true( + all(is.finite(suppressWarnings(MSstats:::.cgSolve( + degenerate_diagonal_matrix, degenerate_rhs, + use_jacobi_preconditioner = TRUE))$solution)), + info = "A zero diagonal entry should not produce a non-finite preconditioned solution" +) diff --git a/inst/tinytest/test_utils_imputation_cg.R b/inst/tinytest/test_utils_imputation_cg.R index 4f57bfc0..8dc8e07a 100644 --- a/inst/tinytest/test_utils_imputation_cg.R +++ b/inst/tinytest/test_utils_imputation_cg.R @@ -85,9 +85,11 @@ make_noisy_censored_input <- function(seed, is_labeled) { dt } -check_solvers_agree <- function(input, tolerance, label) { +check_solvers_agree <- function(input, tolerance, label, + use_jacobi_preconditioner = FALSE) { fit_cholesky <- MSstats:::.fitSurvival(input, 90) - fit_cg <- MSstats:::.fitSurvivalCG(input, 90) + fit_cg <- MSstats:::.fitSurvivalCG( + input, 90, use_jacobi_preconditioner = use_jacobi_preconditioner) matched_names <- names(fit_cholesky$coefficients) expect_equal( @@ -125,3 +127,65 @@ check_solvers_agree( make_noisy_censored_input(seed = 2, is_labeled = FALSE), tolerance = 1e-4, label = "unlabeled, noisy, censored" ) + +# --- the Jacobi-preconditioned solver (aft_solver = "pcg") agrees too ----- + +check_solvers_agree( + make_noisy_censored_input(seed = 1, is_labeled = TRUE), + tolerance = 1e-4, label = "labeled, noisy, censored, jacobi-preconditioned", + use_jacobi_preconditioner = TRUE +) +check_solvers_agree( + make_noisy_censored_input(seed = 2, is_labeled = FALSE), + tolerance = 1e-4, label = "unlabeled, noisy, censored, jacobi-preconditioned", + use_jacobi_preconditioner = TRUE +) + +# --- .fitAFTModel() dispatches to the right solver ------------------------- + +noisy_input <- make_noisy_censored_input(seed = 3, is_labeled = FALSE) + +expect_inherits( + MSstats:::.fitAFTModel(noisy_input, 90, "cholesky"), "survreg", + info = ".fitAFTModel(aft_solver = 'cholesky') should return a survreg fit" +) +expect_true( + is.null(MSstats:::.fitAFTModel(noisy_input, 90, "cholesky")$cg_diagnostics), + info = "the cholesky path should not attach cg_diagnostics" +) +expect_false( + is.null(MSstats:::.fitAFTModel(noisy_input, 90, "cg")$cg_diagnostics), + info = ".fitAFTModel(aft_solver = 'cg') should attach cg_diagnostics" +) +expect_false( + is.null(MSstats:::.fitAFTModel(noisy_input, 90, "pcg")$cg_diagnostics), + info = ".fitAFTModel(aft_solver = 'pcg') should attach cg_diagnostics" +) + +# --- verbose = TRUE logs per-iteration diagnostics, FALSE stays silent ----- + +expect_silent( + MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = FALSE) +) +expect_message( + MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = TRUE), + pattern = "\\[AFT-CG\\] starting fit", + info = "verbose = TRUE should report the problem size at the start of the fit" +) +expect_message( + MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = TRUE), + pattern = "\\[AFT-CG\\] finished", + info = "verbose = TRUE should report a summary once fitting finishes" +) + +# --- cg_diagnostics has one row per Newton iteration actually taken ------- + +fit_with_diagnostics <- MSstats:::.fitSurvivalCG(noisy_input, 90) +expect_equal( + nrow(fit_with_diagnostics$cg_diagnostics), fit_with_diagnostics$iter, + info = "cg_diagnostics should have one row per Newton-Raphson iteration taken" +) +expect_true( + all(fit_with_diagnostics$cg_diagnostics$cg_iterations >= 0), + info = "cg_iterations should be a non-negative count for every Newton iteration" +) diff --git a/man/MSstatsSummarizeSingleLinear.Rd b/man/MSstatsSummarizeSingleLinear.Rd index 0f0056dd..dc6d006e 100644 --- a/man/MSstatsSummarizeSingleLinear.Rd +++ b/man/MSstatsSummarizeSingleLinear.Rd @@ -11,7 +11,8 @@ MSstatsSummarizeSingleLinear( remove50missing, aft_iterations = 90, equal_variances = TRUE, - aft_solver = "cholesky" + aft_solver = "cholesky", + aft_verbose = FALSE ) } \arguments{ @@ -29,7 +30,12 @@ MSstatsSummarizeSingleLinear( \item{aft_solver}{Which linear solve to use for the AFT imputation model's Newton-Raphson step: "cholesky" (default, via -\code{survival::survreg}) or "cg" (conjugate gradient).} +\code{survival::survreg}), "cg" (conjugate gradient), or "pcg" +(conjugate gradient with a Jacobi/inverse-diagonal preconditioner).} + +\item{aft_verbose}{If \code{TRUE} and \code{aft_solver} is "cg" or +"pcg", log per-Newton-iteration conjugate-gradient diagnostics for +every protein fit. See \code{.fitSurvivalCG}'s \code{verbose}.} } \value{ list with protein-level data diff --git a/man/MSstatsSummarizeSingleTMP.Rd b/man/MSstatsSummarizeSingleTMP.Rd index 55038a02..02c5f119 100644 --- a/man/MSstatsSummarizeSingleTMP.Rd +++ b/man/MSstatsSummarizeSingleTMP.Rd @@ -10,7 +10,8 @@ MSstatsSummarizeSingleTMP( censored_symbol, remove50missing, aft_iterations = 90, - aft_solver = "cholesky" + aft_solver = "cholesky", + aft_verbose = FALSE ) } \arguments{ diff --git a/man/MSstatsSummarizeWithSingleCore.Rd b/man/MSstatsSummarizeWithSingleCore.Rd index 278ae8e1..94f4ceb3 100644 --- a/man/MSstatsSummarizeWithSingleCore.Rd +++ b/man/MSstatsSummarizeWithSingleCore.Rd @@ -12,7 +12,8 @@ MSstatsSummarizeWithSingleCore( remove50missing, equal_variance, aft_iterations = 90, - aft_solver = "cholesky" + aft_solver = "cholesky", + aft_verbose = FALSE ) } \arguments{ diff --git a/man/dataProcess.Rd b/man/dataProcess.Rd index 5f306a4e..6859dbbf 100644 --- a/man/dataProcess.Rd +++ b/man/dataProcess.Rd @@ -26,7 +26,8 @@ dataProcess( log_file_path = NULL, numberOfCores = 1, aft_iterations = 90, - aft_solver = "cholesky" + aft_solver = "cholesky", + aft_verbose = FALSE ) } \arguments{ @@ -127,7 +128,16 @@ track progress. Only works for Linux & Mac OS. Default is 1.} model's Newton-Raphson step: "cholesky" (default) delegates to \code{survival::survreg}, which solves it via Cholesky factorization. "cg" solves the same Newton step with a vendored conjugate-gradient -routine instead - an experimental alternative, currently opt-in only.} +routine instead; "pcg" is the same conjugate-gradient routine with a +Jacobi (inverse-diagonal) preconditioner, which can reduce the number +of conjugate-gradient iterations needed. "cg"/"pcg" are experimental +alternatives, currently opt-in only.} + +\item{aft_verbose}{If \code{TRUE} and \code{aft_solver} is "cg" or +"pcg", \code{message()} per-Newton-iteration conjugate-gradient +iteration counts and timing for every protein fit - useful for +evaluating solver time complexity, but produces one block of output +per protein, so leave at the default \code{FALSE} for routine runs.} } \value{ A list containing: diff --git a/man/dot-cgSolve.Rd b/man/dot-cgSolve.Rd index 5d846263..853d7e76 100644 --- a/man/dot-cgSolve.Rd +++ b/man/dot-cgSolve.Rd @@ -10,7 +10,8 @@ gradient} right_hand_side, initial_guess = NULL, relative_tolerance = 1e-08, - max_iterations = 10 * nrow(coefficient_matrix) + max_iterations = 10 * nrow(coefficient_matrix), + use_jacobi_preconditioner = FALSE ) } \arguments{ @@ -24,24 +25,42 @@ gradient/score vector from a Newton step.} to the zero vector.} \item{relative_tolerance}{how small the residual needs to shrink, -relative to the size of \code{right_hand_side}, before iteration stops.} +relative to the size of \code{right_hand_side}, before iteration stops. +Always judged on the true (unpreconditioned) residual, so this means the +same thing whether or not \code{use_jacobi_preconditioner} is set.} \item{max_iterations}{how many conjugate-gradient steps to try before giving up. In exact arithmetic, conjugate gradient converges within \code{nrow(coefficient_matrix)} steps, but rounding error erodes that guarantee as the system grows, so the default allows for several times that many steps.} + +\item{use_jacobi_preconditioner}{if \code{TRUE}, precondition with the +inverse of \code{coefficient_matrix}'s own diagonal - cheap to apply, +and often enough to cut down the number of iterations needed when the +diagonal dominates (as it typically does for an AFT information matrix, +where each parameter's own curvature tends to be much larger than its +cross-terms with the other parameters). Defaults to \code{FALSE}, which +reduces exactly to plain (unpreconditioned) conjugate gradient.} } \value{ -numeric vector solving (approximately) -\code{coefficient_matrix \%*\% solution = right_hand_side}. +a list with: \code{solution}, the numeric vector solving +(approximately) \code{coefficient_matrix \%*\% solution = +right_hand_side}; \code{iterations}, how many conjugate-gradient steps +were actually taken; \code{converged}, whether the residual tolerance +was met; and \code{positive_definite}, whether \code{coefficient_matrix} +behaved as positive definite throughout (a caller can fall back to a +different matrix, e.g. a Gauss-Newton approximation, when this is +\code{FALSE}). } \description{ A minimal, single right-hand-side conjugate gradient solver, used as the Newton-step linear solve in \code{.fitSurvivalCG}. Modeled on \code{lfe::cgsolve}, stripped down to a single dense matrix and a single right-hand-side vector (no multi-column batching, no \code{Matrix}-package -or operator/closure dispatch, no preconditioning - none of which are -needed for the small, dense AFT information matrices this is used on). +or operator/closure dispatch - neither is needed for the small, dense AFT +information matrices this is used on). Optionally applies a Jacobi +(inverse-diagonal) preconditioner, which \code{lfe::cgsolve} does not +support at all. } \keyword{internal} diff --git a/man/dot-fitAFTModel.Rd b/man/dot-fitAFTModel.Rd new file mode 100644 index 00000000..37f8f2b1 --- /dev/null +++ b/man/dot-fitAFTModel.Rd @@ -0,0 +1,36 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils_imputation.R +\name{.fitAFTModel} +\alias{.fitAFTModel} +\title{Fit the AFT imputation model with the requested solver} +\usage{ +.fitAFTModel( + input, + aft_iterations, + aft_solver = "cholesky", + aft_verbose = FALSE +) +} +\arguments{ +\item{input}{data.table, the same shape \code{.fitSurvival} expects.} + +\item{aft_iterations}{maximum number of iterations for AFT model fitting.} + +\item{aft_solver}{"cholesky" (default, via \code{survival::survreg}), +"cg" (conjugate gradient), or "pcg" (conjugate gradient with a +Jacobi/inverse-diagonal preconditioner).} + +\item{aft_verbose}{passed through to \code{.fitSurvivalCG}'s +\code{verbose} when \code{aft_solver} is "cg" or "pcg"; has no effect +for "cholesky".} +} +\value{ +a fitted model of class \code{"survreg"}. +} +\description{ +Shared dispatch used by both \code{MSstatsSummarizeSingleLinear} and +\code{MSstatsSummarizeSingleTMP} so the \code{aft_solver}/ +\code{aft_verbose} logic lives in one place instead of being duplicated +at both call sites. +} +\keyword{internal} diff --git a/man/dot-fitSurvivalCG.Rd b/man/dot-fitSurvivalCG.Rd index 98ebeb0a..0aed5db9 100644 --- a/man/dot-fitSurvivalCG.Rd +++ b/man/dot-fitSurvivalCG.Rd @@ -5,7 +5,13 @@ \title{Fit a Gaussian, left-censored AFT model with a conjugate-gradient Newton step} \usage{ -.fitSurvivalCG(input, aft_iterations, convergence_tolerance = 1e-09) +.fitSurvivalCG( + input, + aft_iterations, + convergence_tolerance = 1e-09, + use_jacobi_preconditioner = FALSE, + verbose = FALSE +) } \arguments{ \item{input}{data.table, the same shape \code{.fitSurvival} expects.} @@ -15,9 +21,28 @@ Newton step} \item{convergence_tolerance}{stop once the change in log-likelihood between iterations falls below this (matches the default \code{rel.tolerance} in \code{survival::survreg.control}).} + +\item{use_jacobi_preconditioner}{if \code{TRUE}, precondition every +conjugate-gradient solve with the inverse of the current information +matrix's own diagonal (see \code{.cgSolve}'s +\code{use_jacobi_preconditioner}). This is what \code{aft_solver = +"pcg"} enables, versus plain conjugate gradient for \code{"cg"}.} + +\item{verbose}{if \code{TRUE}, \code{message()} a line per +Newton-Raphson iteration - conjugate-gradient iterations used, whether +the Gauss-Newton fallback (see below) was needed, elapsed time, and the +resulting log-likelihood - plus a one-line summary once fitting +finishes. Meant for evaluating how solver choice and problem size +trade off against iteration count and wall time, not for routine use +(this fits one protein at a time, so it is easy to generate a line per +protein across a whole \code{dataProcess()} run).} } \value{ -a fitted model of class \code{"survreg"}. +a fitted model of class \code{"survreg"}, with one added field: +\code{cg_diagnostics}, a data.frame with one row per Newton-Raphson +iteration recording the conjugate-gradient iteration counts and timing +described above (populated regardless of \code{verbose}, so it can be +inspected/aggregated programmatically after the fact). } \description{ An alternative to \code{.fitSurvival} for exactly the same imputation From 93aa0c6807f1d236b8516c7d44c28b8166a996c7 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Tue, 8 Sep 2026 21:26:51 -0400 Subject: [PATCH 03/30] fix imputation funneling of parameters into multicore --- R/MSstatsSummarizeWithMultipleCores.R | 15 ++++++++++----- man/MSstatsSummarizeWithMultipleCores.Rd | 2 ++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/R/MSstatsSummarizeWithMultipleCores.R b/R/MSstatsSummarizeWithMultipleCores.R index 1d4678fa..7af087cf 100644 --- a/R/MSstatsSummarizeWithMultipleCores.R +++ b/R/MSstatsSummarizeWithMultipleCores.R @@ -278,7 +278,7 @@ #' @noRd .build_summarize_worker <- function( use_TMP, impute, censored_symbol, remove50missing, - aft_iterations, equal_variance + aft_iterations, equal_variance, aft_solver, aft_verbose ) { unpack_fn <- .unpack_protein_slot use_TMP_ <- use_TMP @@ -287,6 +287,8 @@ remove50missing_ <- remove50missing aft_iterations_ <- aft_iterations equal_variance_ <- equal_variance + aft_solver_ <- aft_solver + aft_verbose_ <- aft_verbose function(record) { meta <- record$meta @@ -294,12 +296,13 @@ result <- if (use_TMP_) { MSstatsSummarizeSingleTMP( protein_dt, impute_, censored_symbol_, - remove50missing_, aft_iterations_) + remove50missing_, aft_iterations_, aft_solver_, aft_verbose_) } else { MSstatsSummarizeSingleLinear( protein_dt, impute_, censored_symbol_, remove50missing_, aft_iterations_, - equal_variances = equal_variance_) + equal_variances = equal_variance_, + aft_solver = aft_solver_, aft_verbose = aft_verbose_) } result } @@ -361,6 +364,8 @@ MSstatsSummarizeWithMultipleCores <- function( equal_variance, numberOfCores = 1L, aft_iterations = 90L, + aft_solver = "cholesky", + aft_verbose = FALSE, verbose = FALSE, BPPARAM = NULL, track_memory = FALSE, @@ -369,7 +374,7 @@ MSstatsSummarizeWithMultipleCores <- function( if (numberOfCores <= 1L && is.null(BPPARAM)) { return(MSstatsSummarizeWithSingleCore( input, method, impute, censored_symbol, - remove50missing, equal_variance, aft_iterations)) + remove50missing, equal_variance, aft_iterations, aft_solver, aft_verbose)) } start_time <- proc.time()[["elapsed"]] @@ -419,7 +424,7 @@ MSstatsSummarizeWithMultipleCores <- function( worker_fn <- .build_summarize_worker( use_TMP, impute, censored_symbol, remove50missing, - aft_iterations, equal_variance) + aft_iterations, equal_variance, aft_solver, aft_verbose) if (is.null(BPPARAM)) { tasks <- if (max_proteins_per_worker > 0L) { diff --git a/man/MSstatsSummarizeWithMultipleCores.Rd b/man/MSstatsSummarizeWithMultipleCores.Rd index b95b1e81..d2574fde 100644 --- a/man/MSstatsSummarizeWithMultipleCores.Rd +++ b/man/MSstatsSummarizeWithMultipleCores.Rd @@ -13,6 +13,8 @@ MSstatsSummarizeWithMultipleCores( equal_variance, numberOfCores = 1L, aft_iterations = 90L, + aft_solver = "cholesky", + aft_verbose = FALSE, verbose = FALSE, BPPARAM = NULL, track_memory = FALSE, From 0f177bfe0142f5b322ae9e3790fe1e43d3396f56 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Fri, 11 Sep 2026 16:12:01 -0400 Subject: [PATCH 04/30] add divergence in residual warning --- R/dataProcess.R | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/R/dataProcess.R b/R/dataProcess.R index 2a97689a..f2f3fe7b 100755 --- a/R/dataProcess.R +++ b/R/dataProcess.R @@ -485,6 +485,8 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, # Flag to track convergence warning converged = TRUE + convergence_messages = character(0) + diverging_warnings = 0L fit_data = if (is_labeled_reference) { single_protein[(!is_labeled_ref), cols, with = FALSE] @@ -496,12 +498,38 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, survival_fit = withCallingHandlers({ .fitAFTModel(fit_data, aft_iterations, aft_solver, aft_verbose) }, warning = function(w) { - if (grepl("converge", conditionMessage(w), ignore.case = TRUE)) { - message("Convergence warning caught: ", conditionMessage(w)) + warning_message = conditionMessage(w) + if (grepl("residual is diverging", warning_message, fixed = TRUE)) { + diverging_warnings <<- diverging_warnings + 1L + } + if (grepl("converge", warning_message, ignore.case = TRUE)) { + convergence_messages <<- c(convergence_messages, + warning_message) converged <<- FALSE } }) + protein_name = as.character(unique(single_protein$PROTEIN))[1] + log_fun = getOption("MSstatsLog") + if (diverging_warnings > 0L) { + msg = paste0("DIVERGING RESIDUAL for protein: ", protein_name, + " (", diverging_warnings, " warning(s))") + message(msg) + if (is.function(log_fun)) { + log_fun("INFO", msg) + } + } + if (!converged) { + msg = paste0("CONVERGENCE WARNING for protein: ", protein_name, + " (", length(convergence_messages), + " warning(s)) - ", + paste(unique(convergence_messages), collapse = " | ")) + message(msg) + if (is.function(log_fun)) { + log_fun("INFO", msg) + } + } + if (converged) { single_protein[, predicted := predict(survival_fit, newdata = .SD)] } else { From 85d93ec6ff62de87112fd736c6f34e8216d4b65f Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Fri, 11 Sep 2026 22:28:53 -0400 Subject: [PATCH 05/30] add verbose logging options for running survreg imputation --- R/dataProcess.R | 13 +++++++------ R/utils_imputation.R | 38 ++++++++++++++++++++++++++++++++------ man/reexports.Rd | 2 +- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/R/dataProcess.R b/R/dataProcess.R index f2f3fe7b..8466c422 100755 --- a/R/dataProcess.R +++ b/R/dataProcess.R @@ -70,9 +70,10 @@ #' Jacobi (inverse-diagonal) preconditioner, which can reduce the number #' of conjugate-gradient iterations needed. "cg"/"pcg" are experimental #' alternatives, currently opt-in only. -#' @param aft_verbose If \code{TRUE} and \code{aft_solver} is "cg" or -#' "pcg", \code{message()} per-Newton-iteration conjugate-gradient -#' iteration counts and timing for every protein fit - useful for +#' @param aft_verbose If \code{TRUE}, \code{message()} diagnostics for +#' every protein fit: problem size and elapsed fitting time for all +#' solvers, plus per-Newton-iteration conjugate-gradient iteration counts +#' and timing when \code{aft_solver} is "cg" or "pcg" - useful for #' evaluating solver time complexity, but produces one block of output #' per protein, so leave at the default \code{FALSE} for routine runs. #' @inheritParams .documentFunction @@ -278,9 +279,9 @@ MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol #' model's Newton-Raphson step: "cholesky" (default, via #' \code{survival::survreg}), "cg" (conjugate gradient), or "pcg" #' (conjugate gradient with a Jacobi/inverse-diagonal preconditioner). -#' @param aft_verbose If \code{TRUE} and \code{aft_solver} is "cg" or -#' "pcg", log per-Newton-iteration conjugate-gradient diagnostics for -#' every protein fit. See \code{.fitSurvivalCG}'s \code{verbose}. +#' @param aft_verbose If \code{TRUE}, log AFT fitting diagnostics for +#' every protein fit. See \code{.fitSurvival}'s and +#' \code{.fitSurvivalCG}'s \code{verbose}. #' #' @return list with protein-level data #' diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 9aceda7d..824d49bf 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -54,13 +54,39 @@ } } +#' @param input data.table with the columns \code{.buildAFTFormula} needs. +#' @param aft_iterations maximum number of iterations for AFT model fitting. +#' @param verbose if \code{TRUE}, \code{message()} the problem size +#' (observations and parameters) before fitting and the wall time the fit +#' took afterwards, mirroring what \code{.fitSurvivalCG}'s \code{verbose} +#' reports. Meant for comparing solvers, not for routine use (this fits +#' one protein at a time). +#' +#' @importFrom stats model.frame model.matrix #' @importFrom survival survreg #' @keywords internal -.fitSurvival = function(input, aft_iterations) { +.fitSurvival = function(input, aft_iterations, verbose = FALSE) { # TODO: set.seed here? set.seed(100) - fit = survreg(.buildAFTFormula(input), data = input, dist = "gaussian", + aft_formula = .buildAFTFormula(input) + if (verbose) { + # survreg builds these internally; rebuilding them here is only + # worth the extra work when the counts are actually reported. + model_frame = model.frame(aft_formula, data = input) + design_matrix = model.matrix(attr(model_frame, "terms"), model_frame) + message(sprintf( + "[AFT-Cholesky] starting fit: %d observations, %d parameters", + nrow(design_matrix), ncol(design_matrix) + 1)) + } + fit_start_time = Sys.time() + fit = survreg(aft_formula, data = input, dist = "gaussian", control = list(maxiter = aft_iterations)) + if (verbose) { + message(sprintf( + "[AFT-Cholesky] finished: %d iterations, %.4f sec", + fit$iter[length(fit$iter)], + as.numeric(Sys.time() - fit_start_time, units = "secs"))) + } fit$y = NULL fit$linear.predictors = NULL fit @@ -544,9 +570,9 @@ #' @param aft_solver "cholesky" (default, via \code{survival::survreg}), #' "cg" (conjugate gradient), or "pcg" (conjugate gradient with a #' Jacobi/inverse-diagonal preconditioner). -#' @param aft_verbose passed through to \code{.fitSurvivalCG}'s -#' \code{verbose} when \code{aft_solver} is "cg" or "pcg"; has no effect -#' for "cholesky". +#' @param aft_verbose passed through to the chosen solver's +#' \code{verbose}: \code{.fitSurvivalCG}'s for "cg"/"pcg", +#' \code{.fitSurvival}'s for "cholesky". #' #' @return a fitted model of class \code{"survreg"}. #' @@ -559,7 +585,7 @@ } else if (aft_solver == "cg") { .fitSurvivalCG(input, aft_iterations, verbose = aft_verbose) } else { - .fitSurvival(input, aft_iterations) + .fitSurvival(input, aft_iterations, verbose = aft_verbose) } } diff --git a/man/reexports.Rd b/man/reexports.Rd index 04f47fc4..eeac4273 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -21,6 +21,6 @@ These objects are imported from other packages. Follow the links below to see their documentation. \describe{ - \item{MSstatsConvert}{\code{\link[MSstatsConvert:DIANNtoMSstatsFormat]{DIANNtoMSstatsFormat()}}, \code{\link[MSstatsConvert:DIAUmpiretoMSstatsFormat]{DIAUmpiretoMSstatsFormat()}}, \code{\link[MSstatsConvert:FragPipetoMSstatsFormat]{FragPipetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MaxQtoMSstatsFormat]{MaxQtoMSstatsFormat()}}, \code{\link[MSstatsConvert:MZMinetoMSstatsFormat]{MZMinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenMStoMSstatsFormat]{OpenMStoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenSWATHtoMSstatsFormat]{OpenSWATHtoMSstatsFormat()}}, \code{\link[MSstatsConvert:PDtoMSstatsFormat]{PDtoMSstatsFormat()}}, \code{\link[MSstatsConvert:ProgenesistoMSstatsFormat]{ProgenesistoMSstatsFormat()}}, \code{\link[MSstatsConvert:SkylinetoMSstatsFormat]{SkylinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:SpectronauttoMSstatsFormat]{SpectronauttoMSstatsFormat()}}} + \item{MSstatsConvert}{\code{\link[MSstatsConvert:DIANNtoMSstatsFormat]{DIANNtoMSstatsFormat()}}, \code{\link[MSstatsConvert:DIAUmpiretoMSstatsFormat]{DIAUmpiretoMSstatsFormat()}}, \code{\link[MSstatsConvert:FragPipetoMSstatsFormat]{FragPipetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MZMinetoMSstatsFormat]{MZMinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MaxQtoMSstatsFormat]{MaxQtoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenMStoMSstatsFormat]{OpenMStoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenSWATHtoMSstatsFormat]{OpenSWATHtoMSstatsFormat()}}, \code{\link[MSstatsConvert:PDtoMSstatsFormat]{PDtoMSstatsFormat()}}, \code{\link[MSstatsConvert:ProgenesistoMSstatsFormat]{ProgenesistoMSstatsFormat()}}, \code{\link[MSstatsConvert:SkylinetoMSstatsFormat]{SkylinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:SpectronauttoMSstatsFormat]{SpectronauttoMSstatsFormat()}}} }} From 1194b969320121fbc4f4d185cd8df05c0df2eedb Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 12 Sep 2026 13:02:28 -0400 Subject: [PATCH 06/30] Get rid of divergence breakpoint in CG --- R/dataProcess.R | 12 ------------ R/utils_cgsolve.R | 14 -------------- 2 files changed, 26 deletions(-) diff --git a/R/dataProcess.R b/R/dataProcess.R index 8466c422..7fcd6766 100755 --- a/R/dataProcess.R +++ b/R/dataProcess.R @@ -487,7 +487,6 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, # Flag to track convergence warning converged = TRUE convergence_messages = character(0) - diverging_warnings = 0L fit_data = if (is_labeled_reference) { single_protein[(!is_labeled_ref), cols, with = FALSE] @@ -500,9 +499,6 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, .fitAFTModel(fit_data, aft_iterations, aft_solver, aft_verbose) }, warning = function(w) { warning_message = conditionMessage(w) - if (grepl("residual is diverging", warning_message, fixed = TRUE)) { - diverging_warnings <<- diverging_warnings + 1L - } if (grepl("converge", warning_message, ignore.case = TRUE)) { convergence_messages <<- c(convergence_messages, warning_message) @@ -512,14 +508,6 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, protein_name = as.character(unique(single_protein$PROTEIN))[1] log_fun = getOption("MSstatsLog") - if (diverging_warnings > 0L) { - msg = paste0("DIVERGING RESIDUAL for protein: ", protein_name, - " (", diverging_warnings, " warning(s))") - message(msg) - if (is.function(log_fun)) { - log_fun("INFO", msg) - } - } if (!converged) { msg = paste0("CONVERGENCE WARNING for protein: ", protein_name, " (", length(convergence_messages), diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index 4384245c..6cd57efe 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -73,7 +73,6 @@ residual_size = sum(residual * residual) residual_dot_preconditioned_residual = sum(residual * preconditioned_residual) - smallest_residual_size_seen = residual_size # Stop once the residual has shrunk far enough, relative to the size of # the right-hand side (falling back to an absolute scale when that size @@ -109,19 +108,6 @@ solution = solution + step_length * search_direction residual = residual - step_length * matrix_times_search_direction new_residual_size = sum(residual * residual) - smallest_residual_size_seen = - min(smallest_residual_size_seen, new_residual_size) - - # If the residual has grown far past its best value so far, the - # iteration is diverging (e.g. because coefficient_matrix is - # ill-conditioned) - give up and return what we have rather than - # loop until max_iterations. - if (iteration > 10 && - new_residual_size > 1e4 * smallest_residual_size_seen) { - warning(".cgSolve: residual is diverging; returning the best ", - "iterate found so far") - break - } # Choose the next search direction so it doesn't undo the progress # made by earlier directions. From 0566a9fbb78af8930785f9c0bba6d2f3bd6c8731 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sun, 13 Sep 2026 17:39:31 -0400 Subject: [PATCH 07/30] Remove the artificial number_of_halvings > 30 hard-abort. Instead, let halving consume the shared iteration budget (aft_iterations), and when that budget runs out, fall back to the last accepted coefficients/log_scale/current_fit rather than discarding everything and reporting a hard failure. --- R/utils_imputation.R | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 824d49bf..8570facc 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -279,7 +279,11 @@ #' result is used. #' #' @param input data.table, the same shape \code{.fitSurvival} expects. -#' @param aft_iterations maximum number of Newton-Raphson iterations. +#' @param aft_iterations maximum number of log-likelihood evaluations the +#' fit may spend. Newton-Raphson iterations and the step-halvings used to +#' recover from an overshooting step share this one budget; once it is +#' exhausted fitting stops and the last accepted coefficients and scale +#' are returned (with a non-convergence warning), rather than failing. #' @param convergence_tolerance stop once the change in log-likelihood #' between iterations falls below this (matches the default #' \code{rel.tolerance} in \code{survival::survreg.control}). @@ -431,9 +435,13 @@ current_log_likelihood = current_fit$log_likelihood number_of_iterations_used = 0 converged = FALSE + iterations_remaining = aft_iterations cg_diagnostics = vector("list", aft_iterations) - for (iteration in seq_len(aft_iterations)) { + iteration = 0 + while (iterations_remaining > 0) { + iteration = iteration + 1 + iterations_remaining = iterations_remaining - 1 number_of_iterations_used = iteration iteration_start_time = Sys.time() @@ -470,20 +478,16 @@ # strategy (survreg6.c) rather than simply rejecting the step # outright. number_of_halvings = 0 - halving_exhausted = FALSE repeat { candidate_fit = evaluate_log_likelihood_and_derivatives( candidate_coefficients, candidate_log_scale) candidate_improves = is_finite_fit(candidate_fit) && candidate_fit$log_likelihood >= current_log_likelihood - if (candidate_improves) { + if (candidate_improves || iterations_remaining <= 0) { break } + iterations_remaining = iterations_remaining - 1 number_of_halvings = number_of_halvings + 1 - if (number_of_halvings > 30) { - halving_exhausted = TRUE - break - } if (number_of_halvings == 1 && (log_scale - candidate_log_scale) > 1.1) { # a single huge drop in scale is the most common cause of @@ -496,7 +500,7 @@ candidate_log_scale = (candidate_log_scale + 2 * log_scale) / 3 } - if (halving_exhausted) { + if (!candidate_improves) { break } @@ -518,8 +522,9 @@ } if (!converged) { - warning("AFT model (CG solver) ran out of iterations and did not ", - "converge") + warning("AFT model (CG solver) used its full iteration budget ", + "without converging; returning the last accepted ", + "coefficients") } cg_diagnostics = do.call( From 8b180ab53fc82c8a59ad5d3b5b608e7f96b90993 Mon Sep 17 00:00:00 2001 From: tonywu1999 Date: Thu, 17 Sep 2026 15:10:39 -0400 Subject: [PATCH 08/30] set number of threads to 1 per core for blas operations for pcg --- DESCRIPTION | 5 +- NAMESPACE | 195 ++++++++++++++------------ R/MSstatsSummarizeWithMultipleCores.R | 2 + man/MSstatsSummarizeSingleLinear.Rd | 6 +- man/dataProcess.Rd | 7 +- man/dot-fitAFTModel.Rd | 6 +- man/dot-fitSurvivalCG.Rd | 6 +- man/reexports.Rd | 2 +- 8 files changed, 130 insertions(+), 99 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index d890bee3..818a948b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -40,7 +40,8 @@ Imports: parallel, rlang, matter, - BiocParallel + BiocParallel, + RhpcBLASctl Suggests: BiocStyle, knitr, @@ -62,4 +63,4 @@ Packaged: 2017-10-20 02:13:12 UTC; meenachoi LinkingTo: Rcpp, RcppArmadillo -Config/roxygen2/version: 8.0.0 +Config/roxygen2/version: 8.1.0 diff --git a/NAMESPACE b/NAMESPACE index 4bb4e353..5f156811 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -55,101 +55,124 @@ import(data.table) import(ggplot2) import(limma) import(lme4) -importFrom(BiocParallel,bpisup) -importFrom(BiocParallel,bplapply) -importFrom(BiocParallel,bpnworkers) -importFrom(BiocParallel,bpprogressbar) -importFrom(BiocParallel,bpstart) -importFrom(BiocParallel,bpstop) +importFrom(BiocParallel, + bpisup, + bplapply, + bpnworkers, + bpprogressbar, + bpstart, + bpstop +) importFrom(MASS,rlm) -importFrom(MSstatsConvert,DIANNtoMSstatsFormat) -importFrom(MSstatsConvert,DIAUmpiretoMSstatsFormat) -importFrom(MSstatsConvert,FragPipetoMSstatsFormat) -importFrom(MSstatsConvert,MSstatsBalancedDesign) -importFrom(MSstatsConvert,MSstatsClean) -importFrom(MSstatsConvert,MSstatsImport) -importFrom(MSstatsConvert,MSstatsLogsSettings) -importFrom(MSstatsConvert,MSstatsMakeAnnotation) -importFrom(MSstatsConvert,MSstatsPreprocess) -importFrom(MSstatsConvert,MZMinetoMSstatsFormat) -importFrom(MSstatsConvert,MaxQtoMSstatsFormat) -importFrom(MSstatsConvert,OpenMStoMSstatsFormat) -importFrom(MSstatsConvert,OpenSWATHtoMSstatsFormat) -importFrom(MSstatsConvert,PDtoMSstatsFormat) -importFrom(MSstatsConvert,ProgenesistoMSstatsFormat) -importFrom(MSstatsConvert,SkylinetoMSstatsFormat) -importFrom(MSstatsConvert,SpectronauttoMSstatsFormat) +importFrom(MSstatsConvert, + DIANNtoMSstatsFormat, + DIAUmpiretoMSstatsFormat, + FragPipetoMSstatsFormat, + MSstatsBalancedDesign, + MSstatsClean, + MSstatsImport, + MSstatsLogsSettings, + MSstatsMakeAnnotation, + MSstatsPreprocess, + MZMinetoMSstatsFormat, + MaxQtoMSstatsFormat, + OpenMStoMSstatsFormat, + OpenSWATHtoMSstatsFormat, + PDtoMSstatsFormat, + ProgenesistoMSstatsFormat, + SkylinetoMSstatsFormat, + SpectronauttoMSstatsFormat +) importFrom(Rcpp,sourceCpp) -importFrom(data.table,as.data.table) -importFrom(data.table,data.table) -importFrom(data.table,fifelse) -importFrom(data.table,melt) -importFrom(data.table,rbindlist) -importFrom(data.table,setDT) -importFrom(data.table,setDTthreads) -importFrom(data.table,uniqueN) +importFrom(RhpcBLASctl,blas_set_num_threads) +importFrom(data.table, + as.data.table, + data.table, + fifelse, + melt, + rbindlist, + setDT, + setDTthreads, + uniqueN +) importFrom(ggrepel,geom_text_repel) importFrom(gplots,heatmap.2) -importFrom(grDevices,dev.off) -importFrom(grDevices,hcl) -importFrom(grDevices,pdf) -importFrom(graphics,axis) -importFrom(graphics,image) -importFrom(graphics,legend) -importFrom(graphics,mtext) -importFrom(graphics,par) -importFrom(graphics,plot) -importFrom(graphics,plot.new) -importFrom(graphics,title) -importFrom(htmltools,div) -importFrom(htmltools,save_html) -importFrom(htmltools,tagList) +importFrom(grDevices, + dev.off, + hcl, + pdf +) +importFrom(graphics, + axis, + image, + legend, + mtext, + par, + plot, + plot.new, + title +) +importFrom(htmltools, + div, + save_html, + tagList +) importFrom(limma,squeezeVar) importFrom(lme4,lmer) importFrom(marray,maPalette) importFrom(matter,SnowfastParam) importFrom(methods,is) -importFrom(parallel,clusterExport) -importFrom(parallel,makeCluster) -importFrom(parallel,parLapply) -importFrom(parallel,stopCluster) -importFrom(plotly,add_trace) -importFrom(plotly,ggplotly) -importFrom(plotly,layout) -importFrom(plotly,plot_ly) -importFrom(plotly,style) -importFrom(plotly,subplot) +importFrom(parallel, + clusterExport, + makeCluster, + parLapply, + stopCluster +) +importFrom(plotly, + add_trace, + ggplotly, + layout, + plot_ly, + style, + subplot +) importFrom(preprocessCore,normalize.quantiles) importFrom(rlang,.data) -importFrom(stats,dist) -importFrom(stats,dnorm) -importFrom(stats,fitted) -importFrom(stats,formula) -importFrom(stats,hclust) -importFrom(stats,lm) -importFrom(stats,lm.fit) -importFrom(stats,loess) -importFrom(stats,median) -importFrom(stats,model.frame) -importFrom(stats,model.matrix) -importFrom(stats,model.response) -importFrom(stats,na.omit) -importFrom(stats,p.adjust) -importFrom(stats,pnorm) -importFrom(stats,predict) -importFrom(stats,qbinom) -importFrom(stats,qnorm) -importFrom(stats,qt) -importFrom(stats,quantile) -importFrom(stats,resid) -importFrom(stats,residuals) -importFrom(stats,sd) -importFrom(stats,vcov) -importFrom(stats,xtabs) -importFrom(survival,Surv) -importFrom(survival,survreg) -importFrom(utils,combn) -importFrom(utils,sessionInfo) -importFrom(utils,setTxtProgressBar) -importFrom(utils,txtProgressBar) +importFrom(stats, + dist, + dnorm, + fitted, + formula, + hclust, + lm, + lm.fit, + loess, + median, + model.frame, + model.matrix, + model.response, + na.omit, + p.adjust, + pnorm, + predict, + qbinom, + qnorm, + qt, + quantile, + resid, + residuals, + sd, + vcov, + xtabs +) +importFrom(survival, + Surv, + survreg +) +importFrom(utils, + combn, + sessionInfo, + setTxtProgressBar, + txtProgressBar +) useDynLib(MSstats, .registration=TRUE) diff --git a/R/MSstatsSummarizeWithMultipleCores.R b/R/MSstatsSummarizeWithMultipleCores.R index 7af087cf..fff63a15 100644 --- a/R/MSstatsSummarizeWithMultipleCores.R +++ b/R/MSstatsSummarizeWithMultipleCores.R @@ -316,9 +316,11 @@ list(worker = i, pid = Sys.getpid(), max_rss_mb = .max_rss_mb()) } +#' @importFrom RhpcBLASctl blas_set_num_threads .warmup_worker <- function(i) { library(MSstats, quietly = TRUE, warn.conflicts = FALSE) data.table::setDTthreads(1) + RhpcBLASctl::blas_set_num_threads(1) NULL } diff --git a/man/MSstatsSummarizeSingleLinear.Rd b/man/MSstatsSummarizeSingleLinear.Rd index dc6d006e..ed60f8c4 100644 --- a/man/MSstatsSummarizeSingleLinear.Rd +++ b/man/MSstatsSummarizeSingleLinear.Rd @@ -33,9 +33,9 @@ model's Newton-Raphson step: "cholesky" (default, via \code{survival::survreg}), "cg" (conjugate gradient), or "pcg" (conjugate gradient with a Jacobi/inverse-diagonal preconditioner).} -\item{aft_verbose}{If \code{TRUE} and \code{aft_solver} is "cg" or -"pcg", log per-Newton-iteration conjugate-gradient diagnostics for -every protein fit. See \code{.fitSurvivalCG}'s \code{verbose}.} +\item{aft_verbose}{If \code{TRUE}, log AFT fitting diagnostics for +every protein fit. See \code{.fitSurvival}'s and +\code{.fitSurvivalCG}'s \code{verbose}.} } \value{ list with protein-level data diff --git a/man/dataProcess.Rd b/man/dataProcess.Rd index 6859dbbf..99c6497d 100644 --- a/man/dataProcess.Rd +++ b/man/dataProcess.Rd @@ -133,9 +133,10 @@ Jacobi (inverse-diagonal) preconditioner, which can reduce the number of conjugate-gradient iterations needed. "cg"/"pcg" are experimental alternatives, currently opt-in only.} -\item{aft_verbose}{If \code{TRUE} and \code{aft_solver} is "cg" or -"pcg", \code{message()} per-Newton-iteration conjugate-gradient -iteration counts and timing for every protein fit - useful for +\item{aft_verbose}{If \code{TRUE}, \code{message()} diagnostics for +every protein fit: problem size and elapsed fitting time for all +solvers, plus per-Newton-iteration conjugate-gradient iteration counts +and timing when \code{aft_solver} is "cg" or "pcg" - useful for evaluating solver time complexity, but produces one block of output per protein, so leave at the default \code{FALSE} for routine runs.} } diff --git a/man/dot-fitAFTModel.Rd b/man/dot-fitAFTModel.Rd index 37f8f2b1..a66883e2 100644 --- a/man/dot-fitAFTModel.Rd +++ b/man/dot-fitAFTModel.Rd @@ -20,9 +20,9 @@ "cg" (conjugate gradient), or "pcg" (conjugate gradient with a Jacobi/inverse-diagonal preconditioner).} -\item{aft_verbose}{passed through to \code{.fitSurvivalCG}'s -\code{verbose} when \code{aft_solver} is "cg" or "pcg"; has no effect -for "cholesky".} +\item{aft_verbose}{passed through to the chosen solver's +\code{verbose}: \code{.fitSurvivalCG}'s for "cg"/"pcg", +\code{.fitSurvival}'s for "cholesky".} } \value{ a fitted model of class \code{"survreg"}. diff --git a/man/dot-fitSurvivalCG.Rd b/man/dot-fitSurvivalCG.Rd index 0aed5db9..38ab07a4 100644 --- a/man/dot-fitSurvivalCG.Rd +++ b/man/dot-fitSurvivalCG.Rd @@ -16,7 +16,11 @@ Newton step} \arguments{ \item{input}{data.table, the same shape \code{.fitSurvival} expects.} -\item{aft_iterations}{maximum number of Newton-Raphson iterations.} +\item{aft_iterations}{maximum number of log-likelihood evaluations the +fit may spend. Newton-Raphson iterations and the step-halvings used to +recover from an overshooting step share this one budget; once it is +exhausted fitting stops and the last accepted coefficients and scale +are returned (with a non-convergence warning), rather than failing.} \item{convergence_tolerance}{stop once the change in log-likelihood between iterations falls below this (matches the default diff --git a/man/reexports.Rd b/man/reexports.Rd index eeac4273..04f47fc4 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -21,6 +21,6 @@ These objects are imported from other packages. Follow the links below to see their documentation. \describe{ - \item{MSstatsConvert}{\code{\link[MSstatsConvert:DIANNtoMSstatsFormat]{DIANNtoMSstatsFormat()}}, \code{\link[MSstatsConvert:DIAUmpiretoMSstatsFormat]{DIAUmpiretoMSstatsFormat()}}, \code{\link[MSstatsConvert:FragPipetoMSstatsFormat]{FragPipetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MZMinetoMSstatsFormat]{MZMinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MaxQtoMSstatsFormat]{MaxQtoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenMStoMSstatsFormat]{OpenMStoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenSWATHtoMSstatsFormat]{OpenSWATHtoMSstatsFormat()}}, \code{\link[MSstatsConvert:PDtoMSstatsFormat]{PDtoMSstatsFormat()}}, \code{\link[MSstatsConvert:ProgenesistoMSstatsFormat]{ProgenesistoMSstatsFormat()}}, \code{\link[MSstatsConvert:SkylinetoMSstatsFormat]{SkylinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:SpectronauttoMSstatsFormat]{SpectronauttoMSstatsFormat()}}} + \item{MSstatsConvert}{\code{\link[MSstatsConvert:DIANNtoMSstatsFormat]{DIANNtoMSstatsFormat()}}, \code{\link[MSstatsConvert:DIAUmpiretoMSstatsFormat]{DIAUmpiretoMSstatsFormat()}}, \code{\link[MSstatsConvert:FragPipetoMSstatsFormat]{FragPipetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MaxQtoMSstatsFormat]{MaxQtoMSstatsFormat()}}, \code{\link[MSstatsConvert:MZMinetoMSstatsFormat]{MZMinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenMStoMSstatsFormat]{OpenMStoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenSWATHtoMSstatsFormat]{OpenSWATHtoMSstatsFormat()}}, \code{\link[MSstatsConvert:PDtoMSstatsFormat]{PDtoMSstatsFormat()}}, \code{\link[MSstatsConvert:ProgenesistoMSstatsFormat]{ProgenesistoMSstatsFormat()}}, \code{\link[MSstatsConvert:SkylinetoMSstatsFormat]{SkylinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:SpectronauttoMSstatsFormat]{SpectronauttoMSstatsFormat()}}} }} From 7bf9309ec27076f4311729ac4b4d1907c39965ad Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 16:03:44 -0400 Subject: [PATCH 09/30] remove dot documentation files, remove test comments --- NAMESPACE | 194 ++++++++++------------- R/utils_cgsolve.R | 1 + R/utils_imputation.R | 5 + inst/tinytest/test_dataProcess.R | 11 -- inst/tinytest/test_utils_cgsolve.R | 17 -- inst/tinytest/test_utils_imputation_cg.R | 22 --- man/dot-aftGaussianDerivatives.Rd | 54 ------- man/dot-buildAFTFormula.Rd | 30 ---- man/dot-cgSolve.Rd | 66 -------- man/dot-fitAFTModel.Rd | 36 ----- man/dot-fitSurvivalCG.Rd | 65 -------- 11 files changed, 92 insertions(+), 409 deletions(-) delete mode 100644 man/dot-aftGaussianDerivatives.Rd delete mode 100644 man/dot-buildAFTFormula.Rd delete mode 100644 man/dot-cgSolve.Rd delete mode 100644 man/dot-fitAFTModel.Rd delete mode 100644 man/dot-fitSurvivalCG.Rd diff --git a/NAMESPACE b/NAMESPACE index 5f156811..80be4853 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -55,124 +55,102 @@ import(data.table) import(ggplot2) import(limma) import(lme4) -importFrom(BiocParallel, - bpisup, - bplapply, - bpnworkers, - bpprogressbar, - bpstart, - bpstop -) +importFrom(BiocParallel,bpisup) +importFrom(BiocParallel,bplapply) +importFrom(BiocParallel,bpnworkers) +importFrom(BiocParallel,bpprogressbar) +importFrom(BiocParallel,bpstart) +importFrom(BiocParallel,bpstop) importFrom(MASS,rlm) -importFrom(MSstatsConvert, - DIANNtoMSstatsFormat, - DIAUmpiretoMSstatsFormat, - FragPipetoMSstatsFormat, - MSstatsBalancedDesign, - MSstatsClean, - MSstatsImport, - MSstatsLogsSettings, - MSstatsMakeAnnotation, - MSstatsPreprocess, - MZMinetoMSstatsFormat, - MaxQtoMSstatsFormat, - OpenMStoMSstatsFormat, - OpenSWATHtoMSstatsFormat, - PDtoMSstatsFormat, - ProgenesistoMSstatsFormat, - SkylinetoMSstatsFormat, - SpectronauttoMSstatsFormat -) +importFrom(MSstatsConvert,DIANNtoMSstatsFormat) +importFrom(MSstatsConvert,DIAUmpiretoMSstatsFormat) +importFrom(MSstatsConvert,FragPipetoMSstatsFormat) +importFrom(MSstatsConvert,MSstatsBalancedDesign) +importFrom(MSstatsConvert,MSstatsClean) +importFrom(MSstatsConvert,MSstatsImport) +importFrom(MSstatsConvert,MSstatsLogsSettings) +importFrom(MSstatsConvert,MSstatsMakeAnnotation) +importFrom(MSstatsConvert,MSstatsPreprocess) +importFrom(MSstatsConvert,MZMinetoMSstatsFormat) +importFrom(MSstatsConvert,MaxQtoMSstatsFormat) +importFrom(MSstatsConvert,OpenMStoMSstatsFormat) +importFrom(MSstatsConvert,OpenSWATHtoMSstatsFormat) +importFrom(MSstatsConvert,PDtoMSstatsFormat) +importFrom(MSstatsConvert,ProgenesistoMSstatsFormat) +importFrom(MSstatsConvert,SkylinetoMSstatsFormat) +importFrom(MSstatsConvert,SpectronauttoMSstatsFormat) importFrom(Rcpp,sourceCpp) importFrom(RhpcBLASctl,blas_set_num_threads) -importFrom(data.table, - as.data.table, - data.table, - fifelse, - melt, - rbindlist, - setDT, - setDTthreads, - uniqueN -) +importFrom(data.table,as.data.table) +importFrom(data.table,data.table) +importFrom(data.table,fifelse) +importFrom(data.table,melt) +importFrom(data.table,rbindlist) +importFrom(data.table,setDT) +importFrom(data.table,setDTthreads) +importFrom(data.table,uniqueN) importFrom(ggrepel,geom_text_repel) importFrom(gplots,heatmap.2) -importFrom(grDevices, - dev.off, - hcl, - pdf -) -importFrom(graphics, - axis, - image, - legend, - mtext, - par, - plot, - plot.new, - title -) -importFrom(htmltools, - div, - save_html, - tagList -) +importFrom(grDevices,dev.off) +importFrom(grDevices,hcl) +importFrom(grDevices,pdf) +importFrom(graphics,axis) +importFrom(graphics,image) +importFrom(graphics,legend) +importFrom(graphics,mtext) +importFrom(graphics,par) +importFrom(graphics,plot) +importFrom(graphics,plot.new) +importFrom(graphics,title) +importFrom(htmltools,div) +importFrom(htmltools,save_html) +importFrom(htmltools,tagList) importFrom(limma,squeezeVar) importFrom(lme4,lmer) importFrom(marray,maPalette) importFrom(matter,SnowfastParam) importFrom(methods,is) -importFrom(parallel, - clusterExport, - makeCluster, - parLapply, - stopCluster -) -importFrom(plotly, - add_trace, - ggplotly, - layout, - plot_ly, - style, - subplot -) +importFrom(parallel,clusterExport) +importFrom(parallel,makeCluster) +importFrom(parallel,parLapply) +importFrom(parallel,stopCluster) +importFrom(plotly,add_trace) +importFrom(plotly,ggplotly) +importFrom(plotly,layout) +importFrom(plotly,plot_ly) +importFrom(plotly,style) +importFrom(plotly,subplot) importFrom(preprocessCore,normalize.quantiles) importFrom(rlang,.data) -importFrom(stats, - dist, - dnorm, - fitted, - formula, - hclust, - lm, - lm.fit, - loess, - median, - model.frame, - model.matrix, - model.response, - na.omit, - p.adjust, - pnorm, - predict, - qbinom, - qnorm, - qt, - quantile, - resid, - residuals, - sd, - vcov, - xtabs -) -importFrom(survival, - Surv, - survreg -) -importFrom(utils, - combn, - sessionInfo, - setTxtProgressBar, - txtProgressBar -) +importFrom(stats,dist) +importFrom(stats,dnorm) +importFrom(stats,fitted) +importFrom(stats,formula) +importFrom(stats,hclust) +importFrom(stats,lm) +importFrom(stats,lm.fit) +importFrom(stats,loess) +importFrom(stats,median) +importFrom(stats,model.frame) +importFrom(stats,model.matrix) +importFrom(stats,model.response) +importFrom(stats,na.omit) +importFrom(stats,p.adjust) +importFrom(stats,pnorm) +importFrom(stats,predict) +importFrom(stats,qbinom) +importFrom(stats,qnorm) +importFrom(stats,qt) +importFrom(stats,quantile) +importFrom(stats,resid) +importFrom(stats,residuals) +importFrom(stats,sd) +importFrom(stats,vcov) +importFrom(stats,xtabs) +importFrom(survival,Surv) +importFrom(survival,survreg) +importFrom(utils,combn) +importFrom(utils,sessionInfo) +importFrom(utils,setTxtProgressBar) +importFrom(utils,txtProgressBar) useDynLib(MSstats, .registration=TRUE) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index 6cd57efe..85597424 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -43,6 +43,7 @@ #' \code{FALSE}). #' #' @keywords internal +#' @noRd .cgSolve = function(coefficient_matrix, right_hand_side, initial_guess = NULL, relative_tolerance = 1e-8, max_iterations = 10 * nrow(coefficient_matrix), diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 8570facc..d16bb52a 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -21,6 +21,7 @@ #' @importFrom data.table uniqueN #' @importFrom survival Surv #' @keywords internal +#' @noRd .buildAFTFormula = function(input) { FEATURE = RUN = NULL @@ -65,6 +66,7 @@ #' @importFrom stats model.frame model.matrix #' @importFrom survival survreg #' @keywords internal +#' @noRd .fitSurvival = function(input, aft_iterations, verbose = FALSE) { # TODO: set.seed here? set.seed(100) @@ -130,6 +132,7 @@ #' #' @importFrom stats dnorm pnorm #' @keywords internal +#' @noRd .aftGaussianDerivatives = function(linear_predictor, log_scale, observed_value, exact_indicator) { scale = exp(log_scale) @@ -309,6 +312,7 @@ #' #' @importFrom stats model.frame model.matrix model.response lm.fit sd #' @keywords internal +#' @noRd .fitSurvivalCG = function(input, aft_iterations, convergence_tolerance = 1e-9, use_jacobi_preconditioner = FALSE, @@ -582,6 +586,7 @@ #' @return a fitted model of class \code{"survreg"}. #' #' @keywords internal +#' @noRd .fitAFTModel = function(input, aft_iterations, aft_solver = "cholesky", aft_verbose = FALSE) { if (aft_solver == "pcg") { diff --git a/inst/tinytest/test_dataProcess.R b/inst/tinytest/test_dataProcess.R index 60df17f7..cceddca5 100644 --- a/inst/tinytest/test_dataProcess.R +++ b/inst/tinytest/test_dataProcess.R @@ -424,17 +424,6 @@ expect_true( info = "MSstatsSummarizeSingleTMP SRM: censored L rows must receive a finite imputed predicted value" ) -# --- Same SRM imputation, but via aft_solver = "cg" ------------------------ -# Same invariants must hold (H never imputed, L gets a finite prediction), -# and the imputed values themselves should closely match the default -# aft_solver = "cholesky" path, since both solve the same Newton step. -# -# make_srm_impute_input()'s uncensored values are an exactly noise-free -# linear function of RUN, which makes the Gaussian scale MLE degenerate -# (unbounded as residuals -> 0). That's fine for the qualitative H/L -# invariant checks above, but not a meaningful numeric comparison between -# solvers, so a little jitter is added here to make the fit well-posed. - make_srm_impute_input_with_noise <- function(seed) { input <- make_srm_impute_input() set.seed(seed) diff --git a/inst/tinytest/test_utils_cgsolve.R b/inst/tinytest/test_utils_cgsolve.R index 902c7c4b..d5fe9ebd 100644 --- a/inst/tinytest/test_utils_cgsolve.R +++ b/inst/tinytest/test_utils_cgsolve.R @@ -1,7 +1,3 @@ -# Tests for .cgSolve(), the vendored conjugate-gradient linear solver used -# as the Newton-step solve in .fitSurvivalCG(). Returns a list: -# solution/iterations/converged/positive_definite. - make_random_spd_matrix <- function(size, seed, ridge = 0.01) { set.seed(seed) random_factor <- matrix(rnorm(size * size), size, size) @@ -32,8 +28,6 @@ for (size in c(2, 5, 10, 30, 80)) { ) } -# --- near-singular system: still returns a finite result, with a warning --- - near_singular_matrix <- make_random_spd_matrix(10, seed = 42) near_singular_matrix[1, ] <- 0 near_singular_matrix[, 1] <- 0 @@ -55,8 +49,6 @@ expect_false( "converged = FALSE and/or positive_definite = FALSE") ) -# --- an initial guess that is already the solution converges immediately --- - exact_matrix <- make_random_spd_matrix(6, seed = 7) set.seed(8) exact_rhs <- rnorm(6) @@ -73,8 +65,6 @@ expect_equal( info = "Starting from the exact solution should take zero iterations" ) -# --- relative_tolerance controls how tightly the system is solved --- - loose_matrix <- make_random_spd_matrix(20, seed = 99) set.seed(100) loose_rhs <- rnorm(20) @@ -92,11 +82,6 @@ expect_true( info = "A tighter relative_tolerance should produce a more accurate solution" ) -# --- Jacobi preconditioner: same answer, fewer or equal iterations -------- -# on a diagonally-dominant system (where a diagonal preconditioner is most -# effective), preconditioned CG should converge in no more iterations than -# plain CG, and to the same solution. - make_diagonally_dominant_matrix <- function(size, seed) { set.seed(seed) matrix_off_diagonal <- matrix(runif(size * size, -0.1, 0.1), size, size) @@ -126,8 +111,6 @@ expect_true( preconditioned_result$iterations, ")") ) -# A degenerate (all-zero) diagonal entry should not blow up the -# preconditioner (falls back to an identity-like scale of 1 for that entry). degenerate_diagonal_matrix <- make_random_spd_matrix(8, seed = 55) degenerate_diagonal_matrix[3, 3] <- 0 set.seed(56) diff --git a/inst/tinytest/test_utils_imputation_cg.R b/inst/tinytest/test_utils_imputation_cg.R index 8dc8e07a..67b0bd5e 100644 --- a/inst/tinytest/test_utils_imputation_cg.R +++ b/inst/tinytest/test_utils_imputation_cg.R @@ -1,15 +1,3 @@ -# Tests that .fitSurvivalCG() - the conjugate-gradient alternative to -# .fitSurvival() - fits the same model, and agrees numerically with it. -# -# The scenarios below reuse the noiseless fixtures from -# test_utils_imputation.R purely to check that .fitSurvivalCG() selects the -# same predictors as .fitSurvival() (via the shared .buildAFTFormula()). -# For numeric agreement on the fitted values themselves, a Gaussian AFT -# model needs actual residual variation to estimate - a noiseless design -# has a degenerate (unbounded) scale MLE, so a second set of fixtures below -# adds realistic noise and left-censoring before comparing coefficients, -# scale, and predictions. - make_surv_labeled_single <- function() { runs <- paste0("R", 1:3) dt <- data.table::rbindlist(list( @@ -46,8 +34,6 @@ make_surv_unlabeled_multi_welldetermined <- function() { dt } -# --- .fitSurvivalCG() selects the same predictors as .fitSurvival() ------- - coef_names <- function(fit) names(coef(fit)) for (make_input in list(make_surv_labeled_single, @@ -61,8 +47,6 @@ for (make_input in list(make_surv_labeled_single, ) } -# --- numeric agreement on realistic (noisy, censored) data ---------------- - make_noisy_censored_input <- function(seed, is_labeled) { set.seed(seed) features <- paste0("F", 1:3) @@ -128,8 +112,6 @@ check_solvers_agree( tolerance = 1e-4, label = "unlabeled, noisy, censored" ) -# --- the Jacobi-preconditioned solver (aft_solver = "pcg") agrees too ----- - check_solvers_agree( make_noisy_censored_input(seed = 1, is_labeled = TRUE), tolerance = 1e-4, label = "labeled, noisy, censored, jacobi-preconditioned", @@ -141,8 +123,6 @@ check_solvers_agree( use_jacobi_preconditioner = TRUE ) -# --- .fitAFTModel() dispatches to the right solver ------------------------- - noisy_input <- make_noisy_censored_input(seed = 3, is_labeled = FALSE) expect_inherits( @@ -162,8 +142,6 @@ expect_false( info = ".fitAFTModel(aft_solver = 'pcg') should attach cg_diagnostics" ) -# --- verbose = TRUE logs per-iteration diagnostics, FALSE stays silent ----- - expect_silent( MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = FALSE) ) diff --git a/man/dot-aftGaussianDerivatives.Rd b/man/dot-aftGaussianDerivatives.Rd deleted file mode 100644 index 768e1dee..00000000 --- a/man/dot-aftGaussianDerivatives.Rd +++ /dev/null @@ -1,54 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils_imputation.R -\name{.aftGaussianDerivatives} -\alias{.aftGaussianDerivatives} -\title{Per-observation log-likelihood and derivatives for a Gaussian AFT model} -\usage{ -.aftGaussianDerivatives( - linear_predictor, - log_scale, - observed_value, - exact_indicator -) -} -\arguments{ -\item{linear_predictor}{current linear predictor -(\code{model_matrix \%*\% coefficients}).} - -\item{log_scale}{current log of the scale parameter.} - -\item{observed_value}{observed value (or, for censored rows, the -detection-limit ceiling substituted in by -\code{.setCensoredByThreshold}).} - -\item{exact_indicator}{\code{1} for an exact/uncensored observation, -\code{0} for one left-censored below \code{observed_value}.} -} -\value{ -a list with the total \code{log_likelihood}, and -per-observation vectors \code{gradient_wrt_linear_predictor}, -\code{second_derivative_wrt_linear_predictor}, -\code{gradient_wrt_log_scale}, \code{second_derivative_wrt_log_scale}, -and \code{cross_derivative} -(d2 log_likelihood / d linear_predictor d log_scale). -} -\description{ -Computes what a Newton-Raphson step needs at the current parameter -guess: the log-likelihood, its first derivative with respect to the -linear predictor and to the log of the scale parameter, and the -corresponding second derivatives - all summed/assembled later into the -score vector and information matrix by \code{.fitSurvivalCG}. This only -covers the two cases MSstats' AFT imputation actually uses: an exact -(uncensored) observation, or one left-censored below a detection-limit -ceiling (\code{Surv(y, cen, type = "left")} with \code{cen == 0}). -} -\details{ -The formulas are transcribed term-for-term from \code{survival}'s own -C implementation (\code{survregc1.c}'s \code{gauss_d} function and its -"exact"/"left censored" cases) rather than re-derived by hand, since a -hand re-derivation is an easy place to introduce a sign error; this -function's correctness is instead checked against numerical -differentiation of the log-likelihood (see -\code{test_utils_imputation_cg.R}). -} -\keyword{internal} diff --git a/man/dot-buildAFTFormula.Rd b/man/dot-buildAFTFormula.Rd deleted file mode 100644 index d387353a..00000000 --- a/man/dot-buildAFTFormula.Rd +++ /dev/null @@ -1,30 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils_imputation.R -\name{.buildAFTFormula} -\alias{.buildAFTFormula} -\title{Decide which predictors go into a single protein's AFT imputation model} -\usage{ -.buildAFTFormula(input) -} -\arguments{ -\item{input}{data.table with columns \code{newABUNDANCE}, \code{cen}, -\code{RUN}, \code{FEATURE}, \code{LABEL}, and (for labeled experiments) -\code{ref_covariate}.} -} -\value{ -a formula whose left side is -\code{Surv(newABUNDANCE, cen, type = "left")}. -} -\description{ -MSstats fits an accelerated-failure-time (AFT) model per protein to -impute left-censored values, and predictors are chosen based on how much -information is actually available: whether this is a labeled (SRM) -experiment with a reference channel (\code{ref_covariate}), whether -there is more than one feature to estimate a \code{FEATURE} effect for, -and whether there are enough uncensored observations to estimate that -effect at all. Both \code{.fitSurvival} (Cholesky-based, via -\code{survival::survreg}) and \code{.fitSurvivalCG} (conjugate-gradient -based) share this selection logic, so the two solvers always fit the -same model and differ only in how the Newton step is solved. -} -\keyword{internal} diff --git a/man/dot-cgSolve.Rd b/man/dot-cgSolve.Rd deleted file mode 100644 index 853d7e76..00000000 --- a/man/dot-cgSolve.Rd +++ /dev/null @@ -1,66 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils_cgsolve.R -\name{.cgSolve} -\alias{.cgSolve} -\title{Solve a symmetric positive (semi-)definite linear system via conjugate -gradient} -\usage{ -.cgSolve( - coefficient_matrix, - right_hand_side, - initial_guess = NULL, - relative_tolerance = 1e-08, - max_iterations = 10 * nrow(coefficient_matrix), - use_jacobi_preconditioner = FALSE -) -} -\arguments{ -\item{coefficient_matrix}{symmetric positive (semi-)definite matrix, -e.g. the Hessian/information matrix from a Newton step.} - -\item{right_hand_side}{vector the system is solved against, e.g. the -gradient/score vector from a Newton step.} - -\item{initial_guess}{optional starting point for the iteration. Defaults -to the zero vector.} - -\item{relative_tolerance}{how small the residual needs to shrink, -relative to the size of \code{right_hand_side}, before iteration stops. -Always judged on the true (unpreconditioned) residual, so this means the -same thing whether or not \code{use_jacobi_preconditioner} is set.} - -\item{max_iterations}{how many conjugate-gradient steps to try before -giving up. In exact arithmetic, conjugate gradient converges within -\code{nrow(coefficient_matrix)} steps, but rounding error erodes that -guarantee as the system grows, so the default allows for several times -that many steps.} - -\item{use_jacobi_preconditioner}{if \code{TRUE}, precondition with the -inverse of \code{coefficient_matrix}'s own diagonal - cheap to apply, -and often enough to cut down the number of iterations needed when the -diagonal dominates (as it typically does for an AFT information matrix, -where each parameter's own curvature tends to be much larger than its -cross-terms with the other parameters). Defaults to \code{FALSE}, which -reduces exactly to plain (unpreconditioned) conjugate gradient.} -} -\value{ -a list with: \code{solution}, the numeric vector solving -(approximately) \code{coefficient_matrix \%*\% solution = -right_hand_side}; \code{iterations}, how many conjugate-gradient steps -were actually taken; \code{converged}, whether the residual tolerance -was met; and \code{positive_definite}, whether \code{coefficient_matrix} -behaved as positive definite throughout (a caller can fall back to a -different matrix, e.g. a Gauss-Newton approximation, when this is -\code{FALSE}). -} -\description{ -A minimal, single right-hand-side conjugate gradient solver, used as the -Newton-step linear solve in \code{.fitSurvivalCG}. Modeled on -\code{lfe::cgsolve}, stripped down to a single dense matrix and a single -right-hand-side vector (no multi-column batching, no \code{Matrix}-package -or operator/closure dispatch - neither is needed for the small, dense AFT -information matrices this is used on). Optionally applies a Jacobi -(inverse-diagonal) preconditioner, which \code{lfe::cgsolve} does not -support at all. -} -\keyword{internal} diff --git a/man/dot-fitAFTModel.Rd b/man/dot-fitAFTModel.Rd deleted file mode 100644 index a66883e2..00000000 --- a/man/dot-fitAFTModel.Rd +++ /dev/null @@ -1,36 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils_imputation.R -\name{.fitAFTModel} -\alias{.fitAFTModel} -\title{Fit the AFT imputation model with the requested solver} -\usage{ -.fitAFTModel( - input, - aft_iterations, - aft_solver = "cholesky", - aft_verbose = FALSE -) -} -\arguments{ -\item{input}{data.table, the same shape \code{.fitSurvival} expects.} - -\item{aft_iterations}{maximum number of iterations for AFT model fitting.} - -\item{aft_solver}{"cholesky" (default, via \code{survival::survreg}), -"cg" (conjugate gradient), or "pcg" (conjugate gradient with a -Jacobi/inverse-diagonal preconditioner).} - -\item{aft_verbose}{passed through to the chosen solver's -\code{verbose}: \code{.fitSurvivalCG}'s for "cg"/"pcg", -\code{.fitSurvival}'s for "cholesky".} -} -\value{ -a fitted model of class \code{"survreg"}. -} -\description{ -Shared dispatch used by both \code{MSstatsSummarizeSingleLinear} and -\code{MSstatsSummarizeSingleTMP} so the \code{aft_solver}/ -\code{aft_verbose} logic lives in one place instead of being duplicated -at both call sites. -} -\keyword{internal} diff --git a/man/dot-fitSurvivalCG.Rd b/man/dot-fitSurvivalCG.Rd deleted file mode 100644 index 38ab07a4..00000000 --- a/man/dot-fitSurvivalCG.Rd +++ /dev/null @@ -1,65 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/utils_imputation.R -\name{.fitSurvivalCG} -\alias{.fitSurvivalCG} -\title{Fit a Gaussian, left-censored AFT model with a conjugate-gradient -Newton step} -\usage{ -.fitSurvivalCG( - input, - aft_iterations, - convergence_tolerance = 1e-09, - use_jacobi_preconditioner = FALSE, - verbose = FALSE -) -} -\arguments{ -\item{input}{data.table, the same shape \code{.fitSurvival} expects.} - -\item{aft_iterations}{maximum number of log-likelihood evaluations the -fit may spend. Newton-Raphson iterations and the step-halvings used to -recover from an overshooting step share this one budget; once it is -exhausted fitting stops and the last accepted coefficients and scale -are returned (with a non-convergence warning), rather than failing.} - -\item{convergence_tolerance}{stop once the change in log-likelihood -between iterations falls below this (matches the default -\code{rel.tolerance} in \code{survival::survreg.control}).} - -\item{use_jacobi_preconditioner}{if \code{TRUE}, precondition every -conjugate-gradient solve with the inverse of the current information -matrix's own diagonal (see \code{.cgSolve}'s -\code{use_jacobi_preconditioner}). This is what \code{aft_solver = -"pcg"} enables, versus plain conjugate gradient for \code{"cg"}.} - -\item{verbose}{if \code{TRUE}, \code{message()} a line per -Newton-Raphson iteration - conjugate-gradient iterations used, whether -the Gauss-Newton fallback (see below) was needed, elapsed time, and the -resulting log-likelihood - plus a one-line summary once fitting -finishes. Meant for evaluating how solver choice and problem size -trade off against iteration count and wall time, not for routine use -(this fits one protein at a time, so it is easy to generate a line per -protein across a whole \code{dataProcess()} run).} -} -\value{ -a fitted model of class \code{"survreg"}, with one added field: -\code{cg_diagnostics}, a data.frame with one row per Newton-Raphson -iteration recording the conjugate-gradient iteration counts and timing -described above (populated regardless of \code{verbose}, so it can be -inspected/aggregated programmatically after the fact). -} -\description{ -An alternative to \code{.fitSurvival} for exactly the same imputation -model (Gaussian accelerated-failure-time regression, left-censoring -only, chosen by the same \code{.buildAFTFormula} both solvers share), -used when \code{aft_solver = "cg"}. It runs the same kind of -Newton-Raphson iteration \code{survival::survreg} does - repeatedly -solving \code{information_matrix \%*\% step = gradient} for the next -set of coefficients - but performs that linear solve with the -conjugate-gradient routine \code{.cgSolve} instead of the Cholesky -factorization \code{survreg} uses internally. The returned object is -classed \code{"survreg"} and carries the fields \code{predict.survreg} -needs, so it is a drop-in replacement anywhere \code{.fitSurvival}'s -result is used. -} -\keyword{internal} From b689cb52fdf63ffd12add6366efabf97c272ba82 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 16:17:45 -0400 Subject: [PATCH 10/30] fix unit tests for test_dataProcess to be easier to read and understand --- inst/tinytest/test_dataProcess.R | 181 +++++++++++++------------------ man/reexports.Rd | 2 +- 2 files changed, 77 insertions(+), 106 deletions(-) diff --git a/inst/tinytest/test_dataProcess.R b/inst/tinytest/test_dataProcess.R index cceddca5..21608d3f 100644 --- a/inst/tinytest/test_dataProcess.R +++ b/inst/tinytest/test_dataProcess.R @@ -347,19 +347,14 @@ expect_equal( ) ) -# MSstatsSummarizeSingleTMP: SRM imputation — H rows must NOT be imputed ------ -# For SRM experiments, H is the normalization reference and must never be -# imputed. Only censored L rows (is_labeled_ref=FALSE) should receive a -# predicted value from the survival model. - -make_srm_impute_input <- function() { - runs <- paste0("R", 1:4) - levels_rc <- c("0", runs) - f1 <- data.table::data.table( +make_srm_imputation_input <- function() { + run_names <- paste0("R", 1:4) + reference_covariate_levels <- c("0", run_names) + feature_one_rows <- data.table::data.table( PROTEIN = "P1", FEATURE = "F1", LABEL = c("H","H","H","H", "L","L","L","L"), - RUN = c(runs, runs), + RUN = c(run_names, run_names), # F1-H-R1 censored (H reference — must NOT be imputed) # F1-L-R2 censored (light peptide — MUST be imputed) newABUNDANCE = c(NA, 10.5, 11.0, 11.5, 14.0, NA, 15.0, 15.5), @@ -367,120 +362,96 @@ make_srm_impute_input <- function() { cen = c(0L, 1L, 1L, 1L, 1L, 0L, 1L, 1L), is_labeled_ref = c(TRUE,TRUE,TRUE,TRUE, FALSE,FALSE,FALSE,FALSE) ) - f2 <- data.table::data.table( + feature_two_rows <- data.table::data.table( PROTEIN = "P1", FEATURE = "F2", LABEL = c("H","H","H","H", "L","L","L","L"), - RUN = c(runs, runs), + RUN = c(run_names, run_names), newABUNDANCE = c(10.0,10.5,11.0,11.5, 14.0,14.5,15.0,15.5), censored = rep(FALSE, 8), cen = rep(1L, 8), is_labeled_ref = c(TRUE,TRUE,TRUE,TRUE, FALSE,FALSE,FALSE,FALSE) ) - dt <- data.table::rbindlist(list(f1, f2)) - dt[, ref_covariate := factor( + srm_input <- data.table::rbindlist(list(feature_one_rows, feature_two_rows)) + srm_input[, ref_covariate := factor( ifelse(is_labeled_ref == FALSE, as.character(RUN), "0"), - levels = levels_rc + levels = reference_covariate_levels )] - dt[, FEATURE := factor(FEATURE)] - dt[, RUN := factor(RUN)] - dt[, n_obs := 4L] - dt[, n_obs_run := 2L] - dt[, ANOMALYSCORES := NA_real_] - dt + srm_input[, FEATURE := factor(FEATURE)] + srm_input[, RUN := factor(RUN)] + srm_input[, n_obs := 4L] + srm_input[, n_obs_run := 2L] + srm_input[, ANOMALYSCORES := NA_real_] + srm_input } -result_srm_imp <- MSstatsSummarizeSingleTMP( - make_srm_impute_input(), - impute = TRUE, - censored_symbol = "NA", - remove50missing = FALSE, - aft_iterations = 90 -) - -survival_srm <- result_srm_imp[[2]] - -# Censored H reference row: predicted must remain NA (not imputed) -h_cens_pred <- survival_srm[ - as.character(FEATURE) == "F1" & - as.character(LABEL) == "H" & - as.character(RUN) == "R1", - predicted -] -expect_true( - length(h_cens_pred) > 0 && all(is.na(h_cens_pred)), - info = "MSstatsSummarizeSingleTMP SRM: censored H rows must NOT receive an imputed predicted value" -) - -# Censored L row: predicted must be a finite imputed value -l_cens_pred <- survival_srm[ - as.character(FEATURE) == "F1" & - as.character(LABEL) == "L" & - as.character(RUN) == "R2", - predicted -] -expect_true( - length(l_cens_pred) > 0 && all(is.finite(l_cens_pred)), - info = "MSstatsSummarizeSingleTMP SRM: censored L rows must receive a finite imputed predicted value" -) - -make_srm_impute_input_with_noise <- function(seed) { - input <- make_srm_impute_input() +make_srm_imputation_input_with_noise <- function(seed) { + input <- make_srm_imputation_input() set.seed(seed) input[cen == 1L, newABUNDANCE := newABUNDANCE + rnorm(.N, sd = 0.01)] input } -result_srm_imp_chol_noisy <- MSstatsSummarizeSingleTMP( - make_srm_impute_input_with_noise(seed = 1), - impute = TRUE, - censored_symbol = "NA", - remove50missing = FALSE, - aft_iterations = 90, - aft_solver = "cholesky" -) -result_srm_imp_cg <- MSstatsSummarizeSingleTMP( - make_srm_impute_input_with_noise(seed = 1), - impute = TRUE, - censored_symbol = "NA", - remove50missing = FALSE, - aft_iterations = 90, - aft_solver = "cg" -) +get_censored_row_predictions <- function(input, aft_solver) { + survival_predictions <- MSstatsSummarizeSingleTMP( + input, + impute = TRUE, + censored_symbol = "NA", + remove50missing = FALSE, + aft_iterations = 90, + aft_solver = aft_solver + )[[2]] + get_feature_one_prediction <- function(label, run) { + survival_predictions[ + as.character(FEATURE) == "F1" & + as.character(LABEL) == label & + as.character(RUN) == run, + predicted + ] + } + list( + censored_heavy = get_feature_one_prediction("H", "R1"), + censored_light = get_feature_one_prediction("L", "R2") + ) +} -survival_srm_chol_noisy <- result_srm_imp_chol_noisy[[2]] -survival_srm_cg <- result_srm_imp_cg[[2]] +expect_heavy_not_imputed_and_light_imputed <- function(predictions, + description) { + expect_true( + length(predictions$censored_heavy) > 0 && + all(is.na(predictions$censored_heavy)), + info = sprintf("MSstatsSummarizeSingleTMP SRM (%s): censored H rows must NOT receive an imputed predicted value", description) + ) + expect_true( + length(predictions$censored_light) > 0 && + all(is.finite(predictions$censored_light)), + info = sprintf("MSstatsSummarizeSingleTMP SRM (%s): censored L rows must receive a finite imputed predicted value", description) + ) +} -h_cens_pred_cg <- survival_srm_cg[ - as.character(FEATURE) == "F1" & - as.character(LABEL) == "H" & - as.character(RUN) == "R1", - predicted -] -expect_true( - length(h_cens_pred_cg) > 0 && all(is.na(h_cens_pred_cg)), - info = "MSstatsSummarizeSingleTMP SRM (aft_solver = cg): censored H rows must NOT receive an imputed predicted value" +aft_solvers <- c("cholesky", "cg", "pcg") +noisy_input_predictions_by_solver <- lapply( + setNames(nm = aft_solvers), + function(solver) { + get_censored_row_predictions( + make_srm_imputation_input_with_noise(seed = 1), solver + ) + } ) -l_cens_pred_cg <- survival_srm_cg[ - as.character(FEATURE) == "F1" & - as.character(LABEL) == "L" & - as.character(RUN) == "R2", - predicted -] -l_cens_pred_chol_noisy <- survival_srm_chol_noisy[ - as.character(FEATURE) == "F1" & - as.character(LABEL) == "L" & - as.character(RUN) == "R2", - predicted -] -expect_true( - length(l_cens_pred_cg) > 0 && all(is.finite(l_cens_pred_cg)), - info = "MSstatsSummarizeSingleTMP SRM (aft_solver = cg): censored L rows must receive a finite imputed predicted value" -) -expect_equal( - l_cens_pred_cg, l_cens_pred_chol_noisy, tolerance = 1e-4, - check.attributes = FALSE, - info = "MSstatsSummarizeSingleTMP SRM: aft_solver = cg should closely match aft_solver = cholesky" -) +for (solver in aft_solvers) { + expect_heavy_not_imputed_and_light_imputed( + noisy_input_predictions_by_solver[[solver]], + sprintf("aft_solver = %s", solver) + ) +} + +for (solver in setdiff(aft_solvers, "cholesky")) { + expect_equal( + noisy_input_predictions_by_solver[[solver]]$censored_light, + noisy_input_predictions_by_solver[["cholesky"]]$censored_light, + tolerance = 1e-6, check.attributes = FALSE, + info = sprintf("MSstatsSummarizeSingleTMP SRM: aft_solver = %s should closely match aft_solver = cholesky", solver) + ) +} diff --git a/man/reexports.Rd b/man/reexports.Rd index 04f47fc4..eeac4273 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -21,6 +21,6 @@ These objects are imported from other packages. Follow the links below to see their documentation. \describe{ - \item{MSstatsConvert}{\code{\link[MSstatsConvert:DIANNtoMSstatsFormat]{DIANNtoMSstatsFormat()}}, \code{\link[MSstatsConvert:DIAUmpiretoMSstatsFormat]{DIAUmpiretoMSstatsFormat()}}, \code{\link[MSstatsConvert:FragPipetoMSstatsFormat]{FragPipetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MaxQtoMSstatsFormat]{MaxQtoMSstatsFormat()}}, \code{\link[MSstatsConvert:MZMinetoMSstatsFormat]{MZMinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenMStoMSstatsFormat]{OpenMStoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenSWATHtoMSstatsFormat]{OpenSWATHtoMSstatsFormat()}}, \code{\link[MSstatsConvert:PDtoMSstatsFormat]{PDtoMSstatsFormat()}}, \code{\link[MSstatsConvert:ProgenesistoMSstatsFormat]{ProgenesistoMSstatsFormat()}}, \code{\link[MSstatsConvert:SkylinetoMSstatsFormat]{SkylinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:SpectronauttoMSstatsFormat]{SpectronauttoMSstatsFormat()}}} + \item{MSstatsConvert}{\code{\link[MSstatsConvert:DIANNtoMSstatsFormat]{DIANNtoMSstatsFormat()}}, \code{\link[MSstatsConvert:DIAUmpiretoMSstatsFormat]{DIAUmpiretoMSstatsFormat()}}, \code{\link[MSstatsConvert:FragPipetoMSstatsFormat]{FragPipetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MZMinetoMSstatsFormat]{MZMinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MaxQtoMSstatsFormat]{MaxQtoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenMStoMSstatsFormat]{OpenMStoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenSWATHtoMSstatsFormat]{OpenSWATHtoMSstatsFormat()}}, \code{\link[MSstatsConvert:PDtoMSstatsFormat]{PDtoMSstatsFormat()}}, \code{\link[MSstatsConvert:ProgenesistoMSstatsFormat]{ProgenesistoMSstatsFormat()}}, \code{\link[MSstatsConvert:SkylinetoMSstatsFormat]{SkylinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:SpectronauttoMSstatsFormat]{SpectronauttoMSstatsFormat()}}} }} From 27e558146b64bd7cc717037a9a33fba71dbd3e0b Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 16:29:37 -0400 Subject: [PATCH 11/30] remove unnecessary parameters for cgsolve --- R/utils_cgsolve.R | 31 +++++++++++----------------- inst/tinytest/test_utils_cgsolve.R | 33 ------------------------------ 2 files changed, 12 insertions(+), 52 deletions(-) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index 85597424..b1561d34 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -10,21 +10,18 @@ #' (inverse-diagonal) preconditioner, which \code{lfe::cgsolve} does not #' support at all. #' +#' Iteration always starts from the zero vector and stops once the true +#' (unpreconditioned) residual has shrunk to \code{1e-8} of the size of +#' \code{right_hand_side}, so the tolerance means the same thing whether or +#' not \code{use_jacobi_preconditioner} is set. In exact arithmetic, +#' conjugate gradient converges within \code{nrow(coefficient_matrix)} +#' steps, but rounding error erodes that guarantee as the system grows, so +#' up to \code{10 * nrow(coefficient_matrix)} steps are allowed. +#' #' @param coefficient_matrix symmetric positive (semi-)definite matrix, #' e.g. the Hessian/information matrix from a Newton step. #' @param right_hand_side vector the system is solved against, e.g. the #' gradient/score vector from a Newton step. -#' @param initial_guess optional starting point for the iteration. Defaults -#' to the zero vector. -#' @param relative_tolerance how small the residual needs to shrink, -#' relative to the size of \code{right_hand_side}, before iteration stops. -#' Always judged on the true (unpreconditioned) residual, so this means the -#' same thing whether or not \code{use_jacobi_preconditioner} is set. -#' @param max_iterations how many conjugate-gradient steps to try before -#' giving up. In exact arithmetic, conjugate gradient converges within -#' \code{nrow(coefficient_matrix)} steps, but rounding error erodes that -#' guarantee as the system grows, so the default allows for several times -#' that many steps. #' @param use_jacobi_preconditioner if \code{TRUE}, precondition with the #' inverse of \code{coefficient_matrix}'s own diagonal - cheap to apply, #' and often enough to cut down the number of iterations needed when the @@ -44,16 +41,12 @@ #' #' @keywords internal #' @noRd -.cgSolve = function(coefficient_matrix, right_hand_side, initial_guess = NULL, - relative_tolerance = 1e-8, - max_iterations = 10 * nrow(coefficient_matrix), +.cgSolve = function(coefficient_matrix, right_hand_side, use_jacobi_preconditioner = FALSE) { number_of_unknowns = nrow(coefficient_matrix) - solution = if (is.null(initial_guess)) { - rep(0, number_of_unknowns) - } else { - initial_guess - } + relative_tolerance = 1e-8 + max_iterations = 10 * number_of_unknowns + solution = rep(0, number_of_unknowns) apply_preconditioner = if (use_jacobi_preconditioner) { diagonal = diag(coefficient_matrix) diff --git a/inst/tinytest/test_utils_cgsolve.R b/inst/tinytest/test_utils_cgsolve.R index d5fe9ebd..71119b0b 100644 --- a/inst/tinytest/test_utils_cgsolve.R +++ b/inst/tinytest/test_utils_cgsolve.R @@ -49,39 +49,6 @@ expect_false( "converged = FALSE and/or positive_definite = FALSE") ) -exact_matrix <- make_random_spd_matrix(6, seed = 7) -set.seed(8) -exact_rhs <- rnorm(6) -exact_answer <- solve(exact_matrix, exact_rhs) - -result_from_exact_start <- MSstats:::.cgSolve( - exact_matrix, exact_rhs, initial_guess = exact_answer) -expect_equal( - result_from_exact_start$solution, exact_answer, tolerance = 1e-8, - info = "Starting from the exact solution should return it unchanged" -) -expect_equal( - result_from_exact_start$iterations, 0, - info = "Starting from the exact solution should take zero iterations" -) - -loose_matrix <- make_random_spd_matrix(20, seed = 99) -set.seed(100) -loose_rhs <- rnorm(20) -exact_loose_answer <- solve(loose_matrix, loose_rhs) - -loose_result <- MSstats:::.cgSolve( - loose_matrix, loose_rhs, relative_tolerance = 1e-2) -tight_result <- MSstats:::.cgSolve( - loose_matrix, loose_rhs, relative_tolerance = 1e-10) - -loose_error <- max(abs(loose_result$solution - exact_loose_answer)) -tight_error <- max(abs(tight_result$solution - exact_loose_answer)) -expect_true( - tight_error < loose_error, - info = "A tighter relative_tolerance should produce a more accurate solution" -) - make_diagonally_dominant_matrix <- function(size, seed) { set.seed(seed) matrix_off_diagonal <- matrix(runif(size * size, -0.1, 0.1), size, size) From ace892ca8aec38f50bacece06d8e5e5d6de30ce2 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 16:45:38 -0400 Subject: [PATCH 12/30] make unit tests of utils_cgsolve more comprehensible --- inst/tinytest/test_utils_cgsolve.R | 112 +++++++++++++++++------------ 1 file changed, 65 insertions(+), 47 deletions(-) diff --git a/inst/tinytest/test_utils_cgsolve.R b/inst/tinytest/test_utils_cgsolve.R index 71119b0b..1b9ceaaa 100644 --- a/inst/tinytest/test_utils_cgsolve.R +++ b/inst/tinytest/test_utils_cgsolve.R @@ -1,90 +1,108 @@ -make_random_spd_matrix <- function(size, seed, ridge = 0.01) { +make_random_solvable_matrix <- function(size, seed, diagonal_boost = 0.01) { + # Builds a random symmetric positive-definite matrix set.seed(seed) - random_factor <- matrix(rnorm(size * size), size, size) - random_factor %*% t(random_factor) + diag(size) * ridge + random_matrix <- matrix(rnorm(size * size), size, size) + random_matrix %*% t(random_matrix) + diag(size) * diagonal_boost } for (size in c(2, 5, 10, 30, 80)) { - coefficient_matrix <- make_random_spd_matrix(size, seed = size) + coefficient_matrix <- make_random_solvable_matrix(size, seed = size) set.seed(size + 1000) right_hand_side <- rnorm(size) - cg_result <- MSstats:::.cgSolve(coefficient_matrix, right_hand_side) + iterative_result <- MSstats:::.cgSolve(coefficient_matrix, right_hand_side) exact_solution <- solve(coefficient_matrix, right_hand_side) expect_equal( - cg_result$solution, exact_solution, tolerance = 1e-6, - info = paste0(".cgSolve should match solve() on a random SPD ", - "system of size ", size) + iterative_result$solution, exact_solution, tolerance = 1e-6, + info = paste0(".cgSolve should give the same answer as solve() on a ", + "random solvable symmetric system of size ", size) ) expect_true( - cg_result$converged && cg_result$positive_definite, - info = paste0("A well-conditioned SPD system of size ", size, - " should report converged/positive_definite = TRUE") + iterative_result$converged && iterative_result$positive_definite, + info = paste0("A well-behaved symmetric system of size ", size, + " should report converged = TRUE and ", + "positive_definite = TRUE") ) expect_true( - cg_result$iterations >= 1 && cg_result$iterations <= size * 10, - info = "iterations should be a small positive count, not the default cap" + iterative_result$iterations >= 1 && + iterative_result$iterations <= size * 10, + info = paste("The number of steps taken should be at least one and", + "should not go over the maximum allowed") ) } -near_singular_matrix <- make_random_spd_matrix(10, seed = 42) -near_singular_matrix[1, ] <- 0 -near_singular_matrix[, 1] <- 0 +make_random_unsolvable_matrix <- function(size, seed) { + unsolvable_matrix <- make_random_solvable_matrix(size, seed) + unsolvable_matrix[1, ] <- 0 + unsolvable_matrix[, 1] <- 0 + unsolvable_matrix +} + +unsolvable_matrix <- make_random_unsolvable_matrix(10, seed = 42) set.seed(43) right_hand_side <- rnorm(10) expect_warning( - singular_result <- MSstats:::.cgSolve(near_singular_matrix, right_hand_side), - info = paste("A singular coefficient_matrix should warn rather than", - "error or hang") + unsolvable_result <- MSstats:::.cgSolve(unsolvable_matrix, right_hand_side), + info = paste("An unsolvable matrix should produce a warning rather than", + "an error or an endless loop") ) expect_true( - all(is.finite(singular_result$solution)), - info = "A singular system should still return a finite (partial) solution" + all(is.finite(unsolvable_result$solution)), + info = paste("An unsolvable system should still return a partial answer", + "made of ordinary finite numbers") ) expect_false( - singular_result$converged && singular_result$positive_definite, - info = paste("A singular coefficient_matrix should signal trouble via", - "converged = FALSE and/or positive_definite = FALSE") + unsolvable_result$converged && unsolvable_result$positive_definite, + info = paste("An unsolvable matrix should be flagged by reporting", + "converged = FALSE, positive_definite = FALSE, or both") ) -make_diagonally_dominant_matrix <- function(size, seed) { +make_large_diagonal_matrix <- function(size, seed) { set.seed(seed) - matrix_off_diagonal <- matrix(runif(size * size, -0.1, 0.1), size, size) - matrix_off_diagonal <- (matrix_off_diagonal + t(matrix_off_diagonal)) / 2 - diag(matrix_off_diagonal) <- 0 - diag(size) * runif(size, 5, 10) + matrix_off_diagonal + small_off_diagonal_entries <- + matrix(runif(size * size, -0.1, 0.1), size, size) + small_off_diagonal_entries <- + (small_off_diagonal_entries + t(small_off_diagonal_entries)) / 2 + diag(small_off_diagonal_entries) <- 0 + diag(size) * runif(size, 5, 10) + small_off_diagonal_entries } -dominant_matrix <- make_diagonally_dominant_matrix(40, seed = 11) +large_diagonal_matrix <- make_large_diagonal_matrix(40, seed = 11) set.seed(12) -dominant_rhs <- rnorm(40) -exact_dominant_answer <- solve(dominant_matrix, dominant_rhs) +large_diagonal_right_hand_side <- rnorm(40) +large_diagonal_exact_solution <- + solve(large_diagonal_matrix, large_diagonal_right_hand_side) -plain_cg_result <- MSstats:::.cgSolve(dominant_matrix, dominant_rhs) -preconditioned_result <- MSstats:::.cgSolve( - dominant_matrix, dominant_rhs, use_jacobi_preconditioner = TRUE) +result_without_scaling <- MSstats:::.cgSolve( + large_diagonal_matrix, large_diagonal_right_hand_side) +result_with_scaling <- MSstats:::.cgSolve( + large_diagonal_matrix, large_diagonal_right_hand_side, + use_jacobi_preconditioner = TRUE) expect_equal( - preconditioned_result$solution, exact_dominant_answer, tolerance = 1e-6, - info = "Preconditioned CG should still match solve() on a diagonally dominant system" + result_with_scaling$solution, large_diagonal_exact_solution, + tolerance = 1e-6, + info = paste("With diagonal scaling turned on, .cgSolve should still", + "give the same answer as solve()") ) expect_true( - preconditioned_result$iterations <= plain_cg_result$iterations, - info = paste("Jacobi preconditioning should not need more iterations", - "than plain CG on a diagonally dominant system (plain =", - plain_cg_result$iterations, ", preconditioned =", - preconditioned_result$iterations, ")") + result_with_scaling$iterations <= result_without_scaling$iterations, + info = paste("Diagonal scaling should not need more steps than", + "no scaling when the diagonal entries are large (without", + "scaling =", result_without_scaling$iterations, + ", with scaling =", result_with_scaling$iterations, ")") ) -degenerate_diagonal_matrix <- make_random_spd_matrix(8, seed = 55) -degenerate_diagonal_matrix[3, 3] <- 0 +zero_diagonal_matrix <- make_random_solvable_matrix(8, seed = 55) +zero_diagonal_matrix[3, 3] <- 0 set.seed(56) -degenerate_rhs <- rnorm(8) +zero_diagonal_right_hand_side <- rnorm(8) expect_true( all(is.finite(suppressWarnings(MSstats:::.cgSolve( - degenerate_diagonal_matrix, degenerate_rhs, + zero_diagonal_matrix, zero_diagonal_right_hand_side, use_jacobi_preconditioner = TRUE))$solution)), - info = "A zero diagonal entry should not produce a non-finite preconditioned solution" + info = paste("A zero on the diagonal should not cause diagonal scaling", + "to return infinite or missing values") ) From d1822a6e138ac5513620ee18ab6b8798f21bce11 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 16:52:56 -0400 Subject: [PATCH 13/30] remove tests_utils_imputation_cg.R --- inst/tinytest/test_utils_imputation.R | 171 +++++++++++++++++++++++ inst/tinytest/test_utils_imputation_cg.R | 169 ---------------------- 2 files changed, 171 insertions(+), 169 deletions(-) delete mode 100644 inst/tinytest/test_utils_imputation_cg.R diff --git a/inst/tinytest/test_utils_imputation.R b/inst/tinytest/test_utils_imputation.R index 03095cf5..b41e21c9 100644 --- a/inst/tinytest/test_utils_imputation.R +++ b/inst/tinytest/test_utils_imputation.R @@ -131,3 +131,174 @@ expect_true( any(grepl("^FEATURE", coef_names(surv_unlabeled_multi_wd))), info = ".fitSurvival unlabeled multi well-determined: FEATURE must appear in coefficients" ) + + +make_surv_labeled_single <- function() { + runs <- paste0("R", 1:3) + dt <- data.table::rbindlist(list( + data.table::data.table( + FEATURE = factor(rep("F1", 9)), + RUN = factor(rep(runs, each = 3)), + LABEL = "H", + newABUNDANCE = seq(10.1, by = 0.1, length.out = 9), + cen = 1L + ), + data.table::data.table( + FEATURE = factor(rep("F1", 9)), + RUN = factor(rep(runs, each = 3)), + LABEL = "L", + newABUNDANCE = seq(14.1, by = 0.1, length.out = 9), + cen = 1L + ) + )) + ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") + dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) + dt +} + +make_surv_unlabeled_multi_welldetermined <- function() { + dt <- data.table::CJ( + FEATURE = paste0("F", 1:3), + RUN = paste0("R", 1:5) + ) + dt[, FEATURE := factor(FEATURE)] + dt[, RUN := factor(RUN)] + dt[, LABEL := "L"] + dt[, newABUNDANCE := seq(10, by = 0.5, length.out = .N)] + dt[, cen := 1L] + dt +} + +coef_names <- function(fit) names(coef(fit)) + +for (make_input in list(make_surv_labeled_single, + make_surv_unlabeled_multi_welldetermined)) { + input <- make_input() + chol_names <- sort(coef_names(MSstats:::.fitSurvival(input, 90))) + cg_names <- sort(coef_names(MSstats:::.fitSurvivalCG(input, 90))) + expect_equal( + cg_names, chol_names, + info = ".fitSurvivalCG must select the same predictors as .fitSurvival" + ) +} + +make_noisy_censored_input <- function(seed, is_labeled) { + set.seed(seed) + features <- paste0("F", 1:3) + runs <- paste0("R", 1:4) + labels <- if (is_labeled) c("H", "L") else "L" + dt <- data.table::CJ(FEATURE = features, RUN = runs, LABEL = labels) + dt[, FEATURE := factor(FEATURE)] + dt[, RUN := factor(RUN)] + dt[, newABUNDANCE := + 10 + as.integer(FEATURE) + as.integer(RUN) * 0.5 + + ifelse(LABEL == "L", 4, 0) + rnorm(.N, sd = 0.7)] + dt[, cen := 1L] + censoring_threshold <- stats::quantile(dt$newABUNDANCE, 0.2) + dt[newABUNDANCE < censoring_threshold, cen := 0L] + dt[cen == 0L, newABUNDANCE := censoring_threshold] + if (is_labeled) { + ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") + dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) + } + dt +} + +check_solvers_agree <- function(input, tolerance, label, + use_jacobi_preconditioner = FALSE) { + fit_cholesky <- MSstats:::.fitSurvival(input, 90) + fit_cg <- MSstats:::.fitSurvivalCG( + input, 90, use_jacobi_preconditioner = use_jacobi_preconditioner) + + matched_names <- names(fit_cholesky$coefficients) + expect_equal( + fit_cg$coefficients[matched_names], + fit_cholesky$coefficients, + tolerance = tolerance, check.attributes = FALSE, + info = paste(label, ": coefficients should match .fitSurvival") + ) + expect_equal( + fit_cg$scale, fit_cholesky$scale, tolerance = tolerance, + check.attributes = FALSE, + info = paste(label, ": scale should match .fitSurvival") + ) + + predictions_cholesky <- predict(fit_cholesky, newdata = input, se.fit = TRUE) + predictions_cg <- predict(fit_cg, newdata = input, se.fit = TRUE) + expect_equal( + predictions_cg$fit, predictions_cholesky$fit, + tolerance = tolerance, check.attributes = FALSE, + info = paste(label, ": predicted values should match .fitSurvival") + ) + expect_equal( + predictions_cg$se.fit, predictions_cholesky$se.fit, + tolerance = tolerance, check.attributes = FALSE, + info = paste(label, ": prediction standard errors should match", + ".fitSurvival") + ) +} + +check_solvers_agree( + make_noisy_censored_input(seed = 1, is_labeled = TRUE), + tolerance = 1e-4, label = "labeled, noisy, censored" +) +check_solvers_agree( + make_noisy_censored_input(seed = 2, is_labeled = FALSE), + tolerance = 1e-4, label = "unlabeled, noisy, censored" +) + +check_solvers_agree( + make_noisy_censored_input(seed = 1, is_labeled = TRUE), + tolerance = 1e-4, label = "labeled, noisy, censored, jacobi-preconditioned", + use_jacobi_preconditioner = TRUE +) +check_solvers_agree( + make_noisy_censored_input(seed = 2, is_labeled = FALSE), + tolerance = 1e-4, label = "unlabeled, noisy, censored, jacobi-preconditioned", + use_jacobi_preconditioner = TRUE +) + +noisy_input <- make_noisy_censored_input(seed = 3, is_labeled = FALSE) + +expect_inherits( + MSstats:::.fitAFTModel(noisy_input, 90, "cholesky"), "survreg", + info = ".fitAFTModel(aft_solver = 'cholesky') should return a survreg fit" +) +expect_true( + is.null(MSstats:::.fitAFTModel(noisy_input, 90, "cholesky")$cg_diagnostics), + info = "the cholesky path should not attach cg_diagnostics" +) +expect_false( + is.null(MSstats:::.fitAFTModel(noisy_input, 90, "cg")$cg_diagnostics), + info = ".fitAFTModel(aft_solver = 'cg') should attach cg_diagnostics" +) +expect_false( + is.null(MSstats:::.fitAFTModel(noisy_input, 90, "pcg")$cg_diagnostics), + info = ".fitAFTModel(aft_solver = 'pcg') should attach cg_diagnostics" +) + +expect_silent( + MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = FALSE) +) +expect_message( + MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = TRUE), + pattern = "\\[AFT-CG\\] starting fit", + info = "verbose = TRUE should report the problem size at the start of the fit" +) +expect_message( + MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = TRUE), + pattern = "\\[AFT-CG\\] finished", + info = "verbose = TRUE should report a summary once fitting finishes" +) + +# --- cg_diagnostics has one row per Newton iteration actually taken ------- + +fit_with_diagnostics <- MSstats:::.fitSurvivalCG(noisy_input, 90) +expect_equal( + nrow(fit_with_diagnostics$cg_diagnostics), fit_with_diagnostics$iter, + info = "cg_diagnostics should have one row per Newton-Raphson iteration taken" +) +expect_true( + all(fit_with_diagnostics$cg_diagnostics$cg_iterations >= 0), + info = "cg_iterations should be a non-negative count for every Newton iteration" +) diff --git a/inst/tinytest/test_utils_imputation_cg.R b/inst/tinytest/test_utils_imputation_cg.R deleted file mode 100644 index 67b0bd5e..00000000 --- a/inst/tinytest/test_utils_imputation_cg.R +++ /dev/null @@ -1,169 +0,0 @@ -make_surv_labeled_single <- function() { - runs <- paste0("R", 1:3) - dt <- data.table::rbindlist(list( - data.table::data.table( - FEATURE = factor(rep("F1", 9)), - RUN = factor(rep(runs, each = 3)), - LABEL = "H", - newABUNDANCE = seq(10.1, by = 0.1, length.out = 9), - cen = 1L - ), - data.table::data.table( - FEATURE = factor(rep("F1", 9)), - RUN = factor(rep(runs, each = 3)), - LABEL = "L", - newABUNDANCE = seq(14.1, by = 0.1, length.out = 9), - cen = 1L - ) - )) - ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") - dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) - dt -} - -make_surv_unlabeled_multi_welldetermined <- function() { - dt <- data.table::CJ( - FEATURE = paste0("F", 1:3), - RUN = paste0("R", 1:5) - ) - dt[, FEATURE := factor(FEATURE)] - dt[, RUN := factor(RUN)] - dt[, LABEL := "L"] - dt[, newABUNDANCE := seq(10, by = 0.5, length.out = .N)] - dt[, cen := 1L] - dt -} - -coef_names <- function(fit) names(coef(fit)) - -for (make_input in list(make_surv_labeled_single, - make_surv_unlabeled_multi_welldetermined)) { - input <- make_input() - chol_names <- sort(coef_names(MSstats:::.fitSurvival(input, 90))) - cg_names <- sort(coef_names(MSstats:::.fitSurvivalCG(input, 90))) - expect_equal( - cg_names, chol_names, - info = ".fitSurvivalCG must select the same predictors as .fitSurvival" - ) -} - -make_noisy_censored_input <- function(seed, is_labeled) { - set.seed(seed) - features <- paste0("F", 1:3) - runs <- paste0("R", 1:4) - labels <- if (is_labeled) c("H", "L") else "L" - dt <- data.table::CJ(FEATURE = features, RUN = runs, LABEL = labels) - dt[, FEATURE := factor(FEATURE)] - dt[, RUN := factor(RUN)] - dt[, newABUNDANCE := - 10 + as.integer(FEATURE) + as.integer(RUN) * 0.5 + - ifelse(LABEL == "L", 4, 0) + rnorm(.N, sd = 0.7)] - dt[, cen := 1L] - censoring_threshold <- stats::quantile(dt$newABUNDANCE, 0.2) - dt[newABUNDANCE < censoring_threshold, cen := 0L] - dt[cen == 0L, newABUNDANCE := censoring_threshold] - if (is_labeled) { - ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") - dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) - } - dt -} - -check_solvers_agree <- function(input, tolerance, label, - use_jacobi_preconditioner = FALSE) { - fit_cholesky <- MSstats:::.fitSurvival(input, 90) - fit_cg <- MSstats:::.fitSurvivalCG( - input, 90, use_jacobi_preconditioner = use_jacobi_preconditioner) - - matched_names <- names(fit_cholesky$coefficients) - expect_equal( - fit_cg$coefficients[matched_names], - fit_cholesky$coefficients, - tolerance = tolerance, check.attributes = FALSE, - info = paste(label, ": coefficients should match .fitSurvival") - ) - expect_equal( - fit_cg$scale, fit_cholesky$scale, tolerance = tolerance, - check.attributes = FALSE, - info = paste(label, ": scale should match .fitSurvival") - ) - - predictions_cholesky <- predict(fit_cholesky, newdata = input, se.fit = TRUE) - predictions_cg <- predict(fit_cg, newdata = input, se.fit = TRUE) - expect_equal( - predictions_cg$fit, predictions_cholesky$fit, - tolerance = tolerance, check.attributes = FALSE, - info = paste(label, ": predicted values should match .fitSurvival") - ) - expect_equal( - predictions_cg$se.fit, predictions_cholesky$se.fit, - tolerance = tolerance, check.attributes = FALSE, - info = paste(label, ": prediction standard errors should match", - ".fitSurvival") - ) -} - -check_solvers_agree( - make_noisy_censored_input(seed = 1, is_labeled = TRUE), - tolerance = 1e-4, label = "labeled, noisy, censored" -) -check_solvers_agree( - make_noisy_censored_input(seed = 2, is_labeled = FALSE), - tolerance = 1e-4, label = "unlabeled, noisy, censored" -) - -check_solvers_agree( - make_noisy_censored_input(seed = 1, is_labeled = TRUE), - tolerance = 1e-4, label = "labeled, noisy, censored, jacobi-preconditioned", - use_jacobi_preconditioner = TRUE -) -check_solvers_agree( - make_noisy_censored_input(seed = 2, is_labeled = FALSE), - tolerance = 1e-4, label = "unlabeled, noisy, censored, jacobi-preconditioned", - use_jacobi_preconditioner = TRUE -) - -noisy_input <- make_noisy_censored_input(seed = 3, is_labeled = FALSE) - -expect_inherits( - MSstats:::.fitAFTModel(noisy_input, 90, "cholesky"), "survreg", - info = ".fitAFTModel(aft_solver = 'cholesky') should return a survreg fit" -) -expect_true( - is.null(MSstats:::.fitAFTModel(noisy_input, 90, "cholesky")$cg_diagnostics), - info = "the cholesky path should not attach cg_diagnostics" -) -expect_false( - is.null(MSstats:::.fitAFTModel(noisy_input, 90, "cg")$cg_diagnostics), - info = ".fitAFTModel(aft_solver = 'cg') should attach cg_diagnostics" -) -expect_false( - is.null(MSstats:::.fitAFTModel(noisy_input, 90, "pcg")$cg_diagnostics), - info = ".fitAFTModel(aft_solver = 'pcg') should attach cg_diagnostics" -) - -expect_silent( - MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = FALSE) -) -expect_message( - MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = TRUE), - pattern = "\\[AFT-CG\\] starting fit", - info = "verbose = TRUE should report the problem size at the start of the fit" -) -expect_message( - MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = TRUE), - pattern = "\\[AFT-CG\\] finished", - info = "verbose = TRUE should report a summary once fitting finishes" -) - -# --- cg_diagnostics has one row per Newton iteration actually taken ------- - -fit_with_diagnostics <- MSstats:::.fitSurvivalCG(noisy_input, 90) -expect_equal( - nrow(fit_with_diagnostics$cg_diagnostics), fit_with_diagnostics$iter, - info = "cg_diagnostics should have one row per Newton-Raphson iteration taken" -) -expect_true( - all(fit_with_diagnostics$cg_diagnostics$cg_iterations >= 0), - info = "cg_iterations should be a non-negative count for every Newton iteration" -) From 12955ff26d0b3c976cf6887a6da89f7aa793a7bb Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 16:55:35 -0400 Subject: [PATCH 14/30] consolidate imputation tests to be simpler and lack redundancy --- inst/tinytest/test_utils_imputation.R | 352 ++++++++------------------ 1 file changed, 106 insertions(+), 246 deletions(-) diff --git a/inst/tinytest/test_utils_imputation.R b/inst/tinytest/test_utils_imputation.R index b41e21c9..cc6a3950 100644 --- a/inst/tinytest/test_utils_imputation.R +++ b/inst/tinytest/test_utils_imputation.R @@ -1,281 +1,143 @@ -make_surv_labeled_single <- function() { - runs <- paste0("R", 1:3) - dt <- data.table::rbindlist(list( - data.table::data.table( - FEATURE = factor(rep("F1", 9)), - RUN = factor(rep(runs, each = 3)), - LABEL = "H", - newABUNDANCE = seq(10.1, by = 0.1, length.out = 9), - cen = 1L - ), - data.table::data.table( - FEATURE = factor(rep("F1", 9)), - RUN = factor(rep(runs, each = 3)), - LABEL = "L", - newABUNDANCE = seq(14.1, by = 0.1, length.out = 9), - cen = 1L - ) - )) - ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") - dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) - dt +add_ref_covariate <- function(dt) { + dt[, ref_covariate := factor(ifelse(LABEL == "L", as.character(RUN), "0"), + levels = c("0", levels(RUN)))] } -make_surv_labeled_multi_welldetermined <- function() { - features <- paste0("F", 1:3) - runs <- paste0("R", 1:4) - dt <- data.table::CJ(FEATURE = features, RUN = runs, LABEL = c("H", "L")) +make_surv_input <- function(n_features, n_runs, is_labeled, n_reps = 1L, + noise_sd = 0.1, censored_fraction = 0, seed = 1) { + set.seed(seed) + dt <- data.table::CJ( + FEATURE = paste0("F", seq_len(n_features)), + RUN = paste0("R", seq_len(n_runs)), + LABEL = if (is_labeled) c("H", "L") else "L", + REPLICATE = seq_len(n_reps) + ) dt[, FEATURE := factor(FEATURE)] dt[, RUN := factor(RUN)] dt[, newABUNDANCE := 10 + as.integer(FEATURE) + as.integer(RUN) * 0.5 + - ifelse(LABEL == "L", 4, 0)] + ifelse(LABEL == "L", 4, 0) + (REPLICATE - 1) * 0.1 + + rnorm(.N, sd = noise_sd)] + dt[, REPLICATE := NULL] dt[, cen := 1L] - ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") - dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) + if (censored_fraction > 0) { + censoring_threshold <- stats::quantile(dt$newABUNDANCE, censored_fraction) + dt[newABUNDANCE < censoring_threshold, + `:=`(cen = 0L, newABUNDANCE = censoring_threshold)] + } + if (is_labeled) add_ref_covariate(dt) dt } make_surv_labeled_underdetermined <- function() { - runs <- c("R1", "R2", "R3") - dt_h <- data.table::data.table( - FEATURE = factor(paste0("F", 1:8)), - RUN = factor(rep_len(runs, 8)), - LABEL = "H", - newABUNDANCE = seq(10, by = 0.5, length.out = 8), + dt <- data.table::data.table( + FEATURE = factor(c(paste0("F", 1:8), "F1")), + RUN = factor(c(rep_len(paste0("R", 1:3), 8), "R1")), + LABEL = c(rep("H", 8), "L"), + newABUNDANCE = c(seq(10, by = 0.5, length.out = 8), 14), cen = 1L ) - dt_l <- data.table::data.table( - FEATURE = factor("F1"), - RUN = factor("R1"), - LABEL = "L", - newABUNDANCE = 14, - cen = 1L - ) - dt <- data.table::rbindlist(list(dt_h, dt_l)) - ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") - dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) - dt -} - -make_surv_unlabeled_single <- function() { - data.table::data.table( - FEATURE = factor(rep("F1", 15)), - RUN = factor(rep(paste0("R", 1:5), each = 3)), - LABEL = "L", - newABUNDANCE = seq(10, by = 0.5, length.out = 15), - cen = 1L - ) -} - -make_surv_unlabeled_multi_welldetermined <- function() { - dt <- data.table::CJ( - FEATURE = paste0("F", 1:3), - RUN = paste0("R", 1:5) - ) - dt[, FEATURE := factor(FEATURE)] - dt[, RUN := factor(RUN)] - dt[, LABEL := "L"] - dt[, newABUNDANCE := seq(10, by = 0.5, length.out = .N)] - dt[, cen := 1L] - dt + add_ref_covariate(dt) } coef_names <- function(fit) names(coef(fit)) -surv_labeled_single <- MSstats:::.fitSurvival(make_surv_labeled_single(), 90) - -expect_true( - any(grepl("ref_covariate", coef_names(surv_labeled_single))), - info = ".fitSurvival labeled single-feature: ref_covariate must appear in coefficients" -) -expect_false( - any(grepl("^FEATURE", coef_names(surv_labeled_single))), - info = ".fitSurvival labeled single-feature: FEATURE must not appear (only one feature)" -) - -surv_labeled_multi_wd <- MSstats:::.fitSurvival(make_surv_labeled_multi_welldetermined(), 90) -expect_true( - any(grepl("ref_covariate", coef_names(surv_labeled_multi_wd))), - info = ".fitSurvival labeled multi well-determined: ref_covariate must appear in coefficients" -) -expect_true( - any(grepl("^FEATURE", coef_names(surv_labeled_multi_wd))), - info = ".fitSurvival labeled multi well-determined: FEATURE must appear in coefficients" -) - -surv_labeled_under <- MSstats:::.fitSurvival(make_surv_labeled_underdetermined(), 90) -expect_true( - any(grepl("ref_covariate", coef_names(surv_labeled_under))), - info = ".fitSurvival labeled underdetermined: ref_covariate must appear in fallback coefficients" -) -expect_false( - any(grepl("^FEATURE", coef_names(surv_labeled_under))), - info = ".fitSurvival labeled underdetermined: FEATURE must not appear in fallback formula" -) - -surv_unlabeled_single <- MSstats:::.fitSurvival(make_surv_unlabeled_single(), 90) -expect_false( - any(grepl("ref_covariate", coef_names(surv_unlabeled_single))), - info = ".fitSurvival unlabeled single-feature: ref_covariate must not appear" -) -expect_false( - any(grepl("^FEATURE", coef_names(surv_unlabeled_single))), - info = ".fitSurvival unlabeled single-feature: FEATURE must not appear (only one feature)" -) - -surv_unlabeled_multi_wd <- MSstats:::.fitSurvival(make_surv_unlabeled_multi_welldetermined(), 90) -expect_false( - any(grepl("ref_covariate", coef_names(surv_unlabeled_multi_wd))), - info = ".fitSurvival unlabeled multi well-determined: ref_covariate must not appear" -) -expect_true( - any(grepl("^FEATURE", coef_names(surv_unlabeled_multi_wd))), - info = ".fitSurvival unlabeled multi well-determined: FEATURE must appear in coefficients" -) - - -make_surv_labeled_single <- function() { - runs <- paste0("R", 1:3) - dt <- data.table::rbindlist(list( - data.table::data.table( - FEATURE = factor(rep("F1", 9)), - RUN = factor(rep(runs, each = 3)), - LABEL = "H", - newABUNDANCE = seq(10.1, by = 0.1, length.out = 9), - cen = 1L - ), - data.table::data.table( - FEATURE = factor(rep("F1", 9)), - RUN = factor(rep(runs, each = 3)), - LABEL = "L", - newABUNDANCE = seq(14.1, by = 0.1, length.out = 9), - cen = 1L - ) - )) - ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") - dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) - dt -} -make_surv_unlabeled_multi_welldetermined <- function() { - dt <- data.table::CJ( - FEATURE = paste0("F", 1:3), - RUN = paste0("R", 1:5) +predictor_cases <- list( + "labeled single-feature" = list( + input = make_surv_input(1, 3, TRUE, n_reps = 3), + has_ref_covariate = TRUE, has_feature = FALSE), + "labeled multi well-determined" = list( + input = make_surv_input(3, 4, TRUE), + has_ref_covariate = TRUE, has_feature = TRUE), + "labeled underdetermined" = list( + input = make_surv_labeled_underdetermined(), + has_ref_covariate = TRUE, has_feature = FALSE), + "unlabeled single-feature" = list( + input = make_surv_input(1, 5, FALSE, n_reps = 3), + has_ref_covariate = FALSE, has_feature = FALSE), + "unlabeled multi well-determined" = list( + input = make_surv_input(3, 5, FALSE), + has_ref_covariate = FALSE, has_feature = TRUE) +) + +for (case_name in names(predictor_cases)) { + case <- predictor_cases[[case_name]] + chol_names <- coef_names(MSstats:::.fitSurvival(case$input, 90)) + expect_equal( + any(grepl("ref_covariate", chol_names)), case$has_ref_covariate, + info = paste(".fitSurvival", case_name, ": ref_covariate in coefficients") ) - dt[, FEATURE := factor(FEATURE)] - dt[, RUN := factor(RUN)] - dt[, LABEL := "L"] - dt[, newABUNDANCE := seq(10, by = 0.5, length.out = .N)] - dt[, cen := 1L] - dt -} - -coef_names <- function(fit) names(coef(fit)) - -for (make_input in list(make_surv_labeled_single, - make_surv_unlabeled_multi_welldetermined)) { - input <- make_input() - chol_names <- sort(coef_names(MSstats:::.fitSurvival(input, 90))) - cg_names <- sort(coef_names(MSstats:::.fitSurvivalCG(input, 90))) expect_equal( - cg_names, chol_names, - info = ".fitSurvivalCG must select the same predictors as .fitSurvival" + any(grepl("^FEATURE", chol_names)), case$has_feature, + info = paste(".fitSurvival", case_name, ": FEATURE in coefficients") + ) + expect_equal( + sort(coef_names(MSstats:::.fitSurvivalCG(case$input, 90))), + sort(chol_names), + info = paste(case_name, ": .fitSurvivalCG must select the same", + "predictors as .fitSurvival") ) } -make_noisy_censored_input <- function(seed, is_labeled) { - set.seed(seed) - features <- paste0("F", 1:3) - runs <- paste0("R", 1:4) - labels <- if (is_labeled) c("H", "L") else "L" - dt <- data.table::CJ(FEATURE = features, RUN = runs, LABEL = labels) - dt[, FEATURE := factor(FEATURE)] - dt[, RUN := factor(RUN)] - dt[, newABUNDANCE := - 10 + as.integer(FEATURE) + as.integer(RUN) * 0.5 + - ifelse(LABEL == "L", 4, 0) + rnorm(.N, sd = 0.7)] - dt[, cen := 1L] - censoring_threshold <- stats::quantile(dt$newABUNDANCE, 0.2) - dt[newABUNDANCE < censoring_threshold, cen := 0L] - dt[cen == 0L, newABUNDANCE := censoring_threshold] - if (is_labeled) { - ref_vals <- ifelse(dt$LABEL == "L", as.character(dt$RUN), "0") - dt[["ref_covariate"]] <- factor(ref_vals, levels = c("0", runs)) - } - dt -} - -check_solvers_agree <- function(input, tolerance, label, - use_jacobi_preconditioner = FALSE) { +check_solvers_agree <- function(input, label, use_jacobi_preconditioner) { fit_cholesky <- MSstats:::.fitSurvival(input, 90) fit_cg <- MSstats:::.fitSurvivalCG( input, 90, use_jacobi_preconditioner = use_jacobi_preconditioner) - - matched_names <- names(fit_cholesky$coefficients) - expect_equal( - fit_cg$coefficients[matched_names], - fit_cholesky$coefficients, - tolerance = tolerance, check.attributes = FALSE, - info = paste(label, ": coefficients should match .fitSurvival") - ) - expect_equal( - fit_cg$scale, fit_cholesky$scale, tolerance = tolerance, - check.attributes = FALSE, - info = paste(label, ": scale should match .fitSurvival") - ) - - predictions_cholesky <- predict(fit_cholesky, newdata = input, se.fit = TRUE) - predictions_cg <- predict(fit_cg, newdata = input, se.fit = TRUE) - expect_equal( - predictions_cg$fit, predictions_cholesky$fit, - tolerance = tolerance, check.attributes = FALSE, - info = paste(label, ": predicted values should match .fitSurvival") - ) - expect_equal( - predictions_cg$se.fit, predictions_cholesky$se.fit, - tolerance = tolerance, check.attributes = FALSE, - info = paste(label, ": prediction standard errors should match", - ".fitSurvival") - ) + summarize_fit <- function(fit) { + predictions <- predict(fit, newdata = input, se.fit = TRUE) + list( + coefficients = fit$coefficients[names(fit_cholesky$coefficients)], + scale = fit$scale, + `predicted values` = predictions$fit, + `prediction standard errors` = predictions$se.fit + ) + } + expected <- summarize_fit(fit_cholesky) + actual <- summarize_fit(fit_cg) + for (quantity in names(expected)) { + expect_equal( + actual[[quantity]], expected[[quantity]], + tolerance = 1e-4, check.attributes = FALSE, + info = paste(label, ":", quantity, "should match .fitSurvival") + ) + } } -check_solvers_agree( - make_noisy_censored_input(seed = 1, is_labeled = TRUE), - tolerance = 1e-4, label = "labeled, noisy, censored" -) -check_solvers_agree( - make_noisy_censored_input(seed = 2, is_labeled = FALSE), - tolerance = 1e-4, label = "unlabeled, noisy, censored" -) - -check_solvers_agree( - make_noisy_censored_input(seed = 1, is_labeled = TRUE), - tolerance = 1e-4, label = "labeled, noisy, censored, jacobi-preconditioned", - use_jacobi_preconditioner = TRUE -) -check_solvers_agree( - make_noisy_censored_input(seed = 2, is_labeled = FALSE), - tolerance = 1e-4, label = "unlabeled, noisy, censored, jacobi-preconditioned", - use_jacobi_preconditioner = TRUE -) +noisy_inputs <- list( + labeled = make_surv_input(3, 4, TRUE, noise_sd = 0.7, + censored_fraction = 0.2, seed = 1), + unlabeled = make_surv_input(3, 4, FALSE, noise_sd = 0.7, + censored_fraction = 0.2, seed = 2) +) +for (input_name in names(noisy_inputs)) { + for (use_jacobi in c(FALSE, TRUE)) { + check_solvers_agree( + noisy_inputs[[input_name]], + label = paste0(input_name, ", noisy, censored", + if (use_jacobi) ", jacobi-preconditioned"), + use_jacobi_preconditioner = use_jacobi + ) + } +} -noisy_input <- make_noisy_censored_input(seed = 3, is_labeled = FALSE) +noisy_input <- make_surv_input(3, 4, FALSE, noise_sd = 0.7, + censored_fraction = 0.2, seed = 3) +fit_cholesky <- MSstats:::.fitAFTModel(noisy_input, 90, "cholesky") expect_inherits( - MSstats:::.fitAFTModel(noisy_input, 90, "cholesky"), "survreg", + fit_cholesky, "survreg", info = ".fitAFTModel(aft_solver = 'cholesky') should return a survreg fit" ) expect_true( - is.null(MSstats:::.fitAFTModel(noisy_input, 90, "cholesky")$cg_diagnostics), + is.null(fit_cholesky$cg_diagnostics), info = "the cholesky path should not attach cg_diagnostics" ) -expect_false( - is.null(MSstats:::.fitAFTModel(noisy_input, 90, "cg")$cg_diagnostics), - info = ".fitAFTModel(aft_solver = 'cg') should attach cg_diagnostics" -) -expect_false( - is.null(MSstats:::.fitAFTModel(noisy_input, 90, "pcg")$cg_diagnostics), - info = ".fitAFTModel(aft_solver = 'pcg') should attach cg_diagnostics" -) +for (aft_solver in c("cg", "pcg")) { + expect_false( + is.null(MSstats:::.fitAFTModel(noisy_input, 90, aft_solver)$cg_diagnostics), + info = paste0(".fitAFTModel(aft_solver = '", aft_solver, + "') should attach cg_diagnostics") + ) +} expect_silent( MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = FALSE) @@ -291,8 +153,6 @@ expect_message( info = "verbose = TRUE should report a summary once fitting finishes" ) -# --- cg_diagnostics has one row per Newton iteration actually taken ------- - fit_with_diagnostics <- MSstats:::.fitSurvivalCG(noisy_input, 90) expect_equal( nrow(fit_with_diagnostics$cg_diagnostics), fit_with_diagnostics$iter, From 6a9e0cef744f9c4d67faaec0ad8fbf2d611b6f3f Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 16:56:27 -0400 Subject: [PATCH 15/30] make utils imputation test file easier to read w.r.t plain english --- inst/tinytest/test_utils_imputation.R | 159 +++++++++++++++----------- 1 file changed, 92 insertions(+), 67 deletions(-) diff --git a/inst/tinytest/test_utils_imputation.R b/inst/tinytest/test_utils_imputation.R index cc6a3950..57b2a58b 100644 --- a/inst/tinytest/test_utils_imputation.R +++ b/inst/tinytest/test_utils_imputation.R @@ -1,87 +1,102 @@ -add_ref_covariate <- function(dt) { - dt[, ref_covariate := factor(ifelse(LABEL == "L", as.character(RUN), "0"), - levels = c("0", levels(RUN)))] +number_of_iterations <- 90 + +add_reference_covariate <- function(input) { + input[, ref_covariate := factor( + ifelse(LABEL == "L", as.character(RUN), "0"), + levels = c("0", levels(RUN)))] } -make_surv_input <- function(n_features, n_runs, is_labeled, n_reps = 1L, - noise_sd = 0.1, censored_fraction = 0, seed = 1) { +make_survival_input <- function(number_of_features, number_of_runs, is_labeled, + number_of_replicates = 1L, + noise_standard_deviation = 0.1, + censored_fraction = 0, seed = 1) { set.seed(seed) - dt <- data.table::CJ( - FEATURE = paste0("F", seq_len(n_features)), - RUN = paste0("R", seq_len(n_runs)), + input <- data.table::CJ( + FEATURE = paste0("F", seq_len(number_of_features)), + RUN = paste0("R", seq_len(number_of_runs)), LABEL = if (is_labeled) c("H", "L") else "L", - REPLICATE = seq_len(n_reps) + REPLICATE = seq_len(number_of_replicates) ) - dt[, FEATURE := factor(FEATURE)] - dt[, RUN := factor(RUN)] - dt[, newABUNDANCE := 10 + as.integer(FEATURE) + as.integer(RUN) * 0.5 + - ifelse(LABEL == "L", 4, 0) + (REPLICATE - 1) * 0.1 + - rnorm(.N, sd = noise_sd)] - dt[, REPLICATE := NULL] - dt[, cen := 1L] + input[, FEATURE := factor(FEATURE)] + input[, RUN := factor(RUN)] + input[, newABUNDANCE := 10 + as.integer(FEATURE) + as.integer(RUN) * 0.5 + + ifelse(LABEL == "L", 4, 0) + (REPLICATE - 1) * 0.1 + + rnorm(.N, sd = noise_standard_deviation)] + input[, REPLICATE := NULL] + input[, cen := 1L] if (censored_fraction > 0) { - censoring_threshold <- stats::quantile(dt$newABUNDANCE, censored_fraction) - dt[newABUNDANCE < censoring_threshold, - `:=`(cen = 0L, newABUNDANCE = censoring_threshold)] + censoring_threshold <- stats::quantile(input$newABUNDANCE, + censored_fraction) + input[newABUNDANCE < censoring_threshold, + `:=`(cen = 0L, newABUNDANCE = censoring_threshold)] } - if (is_labeled) add_ref_covariate(dt) - dt + if (is_labeled) add_reference_covariate(input) + input } -make_surv_labeled_underdetermined <- function() { - dt <- data.table::data.table( +make_underdetermined_labeled_input <- function() { + input <- data.table::data.table( FEATURE = factor(c(paste0("F", 1:8), "F1")), RUN = factor(c(rep_len(paste0("R", 1:3), 8), "R1")), LABEL = c(rep("H", 8), "L"), newABUNDANCE = c(seq(10, by = 0.5, length.out = 8), 14), cen = 1L ) - add_ref_covariate(dt) + add_reference_covariate(input) } -coef_names <- function(fit) names(coef(fit)) +coefficient_names <- function(fit) names(coef(fit)) predictor_cases <- list( "labeled single-feature" = list( - input = make_surv_input(1, 3, TRUE, n_reps = 3), - has_ref_covariate = TRUE, has_feature = FALSE), - "labeled multi well-determined" = list( - input = make_surv_input(3, 4, TRUE), - has_ref_covariate = TRUE, has_feature = TRUE), + input = make_survival_input(number_of_features = 1, number_of_runs = 3, + is_labeled = TRUE, number_of_replicates = 3), + has_reference_covariate = TRUE, has_feature = FALSE), + "labeled multi-feature well-determined" = list( + input = make_survival_input(number_of_features = 3, number_of_runs = 4, + is_labeled = TRUE), + has_reference_covariate = TRUE, has_feature = TRUE), "labeled underdetermined" = list( - input = make_surv_labeled_underdetermined(), - has_ref_covariate = TRUE, has_feature = FALSE), + input = make_underdetermined_labeled_input(), + has_reference_covariate = TRUE, has_feature = FALSE), "unlabeled single-feature" = list( - input = make_surv_input(1, 5, FALSE, n_reps = 3), - has_ref_covariate = FALSE, has_feature = FALSE), - "unlabeled multi well-determined" = list( - input = make_surv_input(3, 5, FALSE), - has_ref_covariate = FALSE, has_feature = TRUE) + input = make_survival_input(number_of_features = 1, number_of_runs = 5, + is_labeled = FALSE, number_of_replicates = 3), + has_reference_covariate = FALSE, has_feature = FALSE), + "unlabeled multi-feature well-determined" = list( + input = make_survival_input(number_of_features = 3, number_of_runs = 5, + is_labeled = FALSE), + has_reference_covariate = FALSE, has_feature = TRUE) ) for (case_name in names(predictor_cases)) { case <- predictor_cases[[case_name]] - chol_names <- coef_names(MSstats:::.fitSurvival(case$input, 90)) + cholesky_coefficient_names <- coefficient_names( + MSstats:::.fitSurvival(case$input, number_of_iterations)) expect_equal( - any(grepl("ref_covariate", chol_names)), case$has_ref_covariate, - info = paste(".fitSurvival", case_name, ": ref_covariate in coefficients") + any(grepl("ref_covariate", cholesky_coefficient_names)), + case$has_reference_covariate, + info = paste(".fitSurvival", case_name, + ": reference covariate in coefficients") ) expect_equal( - any(grepl("^FEATURE", chol_names)), case$has_feature, + any(grepl("^FEATURE", cholesky_coefficient_names)), case$has_feature, info = paste(".fitSurvival", case_name, ": FEATURE in coefficients") ) expect_equal( - sort(coef_names(MSstats:::.fitSurvivalCG(case$input, 90))), - sort(chol_names), - info = paste(case_name, ": .fitSurvivalCG must select the same", - "predictors as .fitSurvival") + sort(coefficient_names( + MSstats:::.fitSurvivalCG(case$input, number_of_iterations))), + sort(cholesky_coefficient_names), + info = paste(case_name, ": the conjugate gradient solver must select", + "the same predictors as the Cholesky solver") ) } check_solvers_agree <- function(input, label, use_jacobi_preconditioner) { - fit_cholesky <- MSstats:::.fitSurvival(input, 90) - fit_cg <- MSstats:::.fitSurvivalCG( - input, 90, use_jacobi_preconditioner = use_jacobi_preconditioner) + fit_cholesky <- MSstats:::.fitSurvival(input, number_of_iterations) + fit_conjugate_gradient <- MSstats:::.fitSurvivalCG( + input, number_of_iterations, + use_jacobi_preconditioner = use_jacobi_preconditioner) summarize_fit <- function(fit) { predictions <- predict(fit, newdata = input, se.fit = TRUE) list( @@ -92,73 +107,83 @@ check_solvers_agree <- function(input, label, use_jacobi_preconditioner) { ) } expected <- summarize_fit(fit_cholesky) - actual <- summarize_fit(fit_cg) + actual <- summarize_fit(fit_conjugate_gradient) for (quantity in names(expected)) { expect_equal( actual[[quantity]], expected[[quantity]], tolerance = 1e-4, check.attributes = FALSE, - info = paste(label, ":", quantity, "should match .fitSurvival") + info = paste(label, ":", quantity, + "should match the Cholesky solver") ) } } +make_noisy_censored_input <- function(is_labeled, seed) { + make_survival_input(number_of_features = 3, number_of_runs = 4, + is_labeled = is_labeled, + noise_standard_deviation = 0.7, + censored_fraction = 0.2, seed = seed) +} + noisy_inputs <- list( - labeled = make_surv_input(3, 4, TRUE, noise_sd = 0.7, - censored_fraction = 0.2, seed = 1), - unlabeled = make_surv_input(3, 4, FALSE, noise_sd = 0.7, - censored_fraction = 0.2, seed = 2) + labeled = make_noisy_censored_input(is_labeled = TRUE, seed = 1), + unlabeled = make_noisy_censored_input(is_labeled = FALSE, seed = 2) ) for (input_name in names(noisy_inputs)) { - for (use_jacobi in c(FALSE, TRUE)) { + for (use_jacobi_preconditioner in c(FALSE, TRUE)) { check_solvers_agree( noisy_inputs[[input_name]], label = paste0(input_name, ", noisy, censored", - if (use_jacobi) ", jacobi-preconditioned"), - use_jacobi_preconditioner = use_jacobi + if (use_jacobi_preconditioner) + ", with Jacobi preconditioner"), + use_jacobi_preconditioner = use_jacobi_preconditioner ) } } -noisy_input <- make_surv_input(3, 4, FALSE, noise_sd = 0.7, - censored_fraction = 0.2, seed = 3) +noisy_input <- make_noisy_censored_input(is_labeled = FALSE, seed = 3) -fit_cholesky <- MSstats:::.fitAFTModel(noisy_input, 90, "cholesky") +fit_cholesky <- MSstats:::.fitAFTModel(noisy_input, number_of_iterations, + "cholesky") expect_inherits( fit_cholesky, "survreg", info = ".fitAFTModel(aft_solver = 'cholesky') should return a survreg fit" ) expect_true( is.null(fit_cholesky$cg_diagnostics), - info = "the cholesky path should not attach cg_diagnostics" + info = "the Cholesky solver should not attach cg_diagnostics" ) for (aft_solver in c("cg", "pcg")) { + fit <- MSstats:::.fitAFTModel(noisy_input, number_of_iterations, aft_solver) expect_false( - is.null(MSstats:::.fitAFTModel(noisy_input, 90, aft_solver)$cg_diagnostics), + is.null(fit$cg_diagnostics), info = paste0(".fitAFTModel(aft_solver = '", aft_solver, "') should attach cg_diagnostics") ) } expect_silent( - MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = FALSE) + MSstats:::.fitSurvivalCG(noisy_input, number_of_iterations, verbose = FALSE) ) expect_message( - MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = TRUE), + MSstats:::.fitSurvivalCG(noisy_input, number_of_iterations, verbose = TRUE), pattern = "\\[AFT-CG\\] starting fit", info = "verbose = TRUE should report the problem size at the start of the fit" ) expect_message( - MSstats:::.fitSurvivalCG(noisy_input, 90, verbose = TRUE), + MSstats:::.fitSurvivalCG(noisy_input, number_of_iterations, verbose = TRUE), pattern = "\\[AFT-CG\\] finished", info = "verbose = TRUE should report a summary once fitting finishes" ) -fit_with_diagnostics <- MSstats:::.fitSurvivalCG(noisy_input, 90) +fit_with_diagnostics <- MSstats:::.fitSurvivalCG(noisy_input, + number_of_iterations) expect_equal( nrow(fit_with_diagnostics$cg_diagnostics), fit_with_diagnostics$iter, info = "cg_diagnostics should have one row per Newton-Raphson iteration taken" ) expect_true( all(fit_with_diagnostics$cg_diagnostics$cg_iterations >= 0), - info = "cg_iterations should be a non-negative count for every Newton iteration" + info = paste("the conjugate gradient iteration count should be", + "non-negative for every Newton iteration") ) From 9749a9edf7f7c9289bb01a0daa1a9fae1d4ee971 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 17:06:52 -0400 Subject: [PATCH 16/30] make utils_imputation tests more comprehensible --- inst/tinytest/test_utils_imputation.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/inst/tinytest/test_utils_imputation.R b/inst/tinytest/test_utils_imputation.R index 57b2a58b..6a0eda78 100644 --- a/inst/tinytest/test_utils_imputation.R +++ b/inst/tinytest/test_utils_imputation.R @@ -34,7 +34,7 @@ make_survival_input <- function(number_of_features, number_of_runs, is_labeled, input } -make_underdetermined_labeled_input <- function() { +make_labeled_input_with_too_few_observations <- function() { input <- data.table::data.table( FEATURE = factor(c(paste0("F", 1:8), "F1")), RUN = factor(c(rep_len(paste0("R", 1:3), 8), "R1")), @@ -52,18 +52,18 @@ predictor_cases <- list( input = make_survival_input(number_of_features = 1, number_of_runs = 3, is_labeled = TRUE, number_of_replicates = 3), has_reference_covariate = TRUE, has_feature = FALSE), - "labeled multi-feature well-determined" = list( + "labeled multi-feature, enough observations" = list( input = make_survival_input(number_of_features = 3, number_of_runs = 4, is_labeled = TRUE), has_reference_covariate = TRUE, has_feature = TRUE), - "labeled underdetermined" = list( - input = make_underdetermined_labeled_input(), + "labeled multi-feature, too few observations" = list( + input = make_labeled_input_with_too_few_observations(), has_reference_covariate = TRUE, has_feature = FALSE), "unlabeled single-feature" = list( input = make_survival_input(number_of_features = 1, number_of_runs = 5, is_labeled = FALSE, number_of_replicates = 3), has_reference_covariate = FALSE, has_feature = FALSE), - "unlabeled multi-feature well-determined" = list( + "unlabeled multi-feature, enough observations" = list( input = make_survival_input(number_of_features = 3, number_of_runs = 5, is_labeled = FALSE), has_reference_covariate = FALSE, has_feature = TRUE) @@ -142,7 +142,6 @@ for (input_name in names(noisy_inputs)) { } noisy_input <- make_noisy_censored_input(is_labeled = FALSE, seed = 3) - fit_cholesky <- MSstats:::.fitAFTModel(noisy_input, number_of_iterations, "cholesky") expect_inherits( @@ -176,6 +175,7 @@ expect_message( info = "verbose = TRUE should report a summary once fitting finishes" ) + fit_with_diagnostics <- MSstats:::.fitSurvivalCG(noisy_input, number_of_iterations) expect_equal( From e148a8f026a758c31855ea265e726bcacee335b3 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 17:13:55 -0400 Subject: [PATCH 17/30] adjust coderabbit comments --- R/MSstatsSummarizeWithMultipleCores.R | 9 +++++++ R/dataProcess.R | 5 +++- R/utils_checks.R | 9 +++++++ R/utils_imputation.R | 1 + inst/tinytest/test_dataProcess.R | 33 ++++++++++++++++++++++++ inst/tinytest/test_utils_imputation.R | 5 ++++ man/MSstatsSummarizeSingleTMP.Rd | 10 +++++++ man/MSstatsSummarizeWithMultipleCores.Rd | 10 +++++++ man/MSstatsSummarizeWithSingleCore.Rd | 10 +++++++ man/reexports.Rd | 2 +- 10 files changed, 92 insertions(+), 2 deletions(-) diff --git a/R/MSstatsSummarizeWithMultipleCores.R b/R/MSstatsSummarizeWithMultipleCores.R index fff63a15..215771e1 100644 --- a/R/MSstatsSummarizeWithMultipleCores.R +++ b/R/MSstatsSummarizeWithMultipleCores.R @@ -335,6 +335,14 @@ #' @param equal_variance only for method = "linear"; assume equal variance among feature intensities #' @param numberOfCores number of cores for parallel processing (Linux/Mac only) #' @param aft_iterations number of AFT model iterations +#' @param aft_solver only used when impute = TRUE; linear solve +#' used in the AFT imputation model's Newton-Raphson step: "cholesky" +#' (default, via \code{survival::survreg}), "cg" (conjugate gradient), or +#' "pcg" (conjugate gradient with a Jacobi/inverse-diagonal preconditioner). +#' "cg"/"pcg" are experimental. +#' @param aft_verbose if \code{TRUE}, \code{message()} AFT fitting diagnostics +#' (problem size, elapsed time, and for "cg"/"pcg" per-Newton-iteration +#' conjugate-gradient counts) for every protein fit. Default \code{FALSE}. #' @param verbose whether to print verbose output #' @param BPPARAM optional \code{BiocParallelParam} instance #' @param track_memory whether to report per-worker maximum RSS memory usage. @@ -373,6 +381,7 @@ MSstatsSummarizeWithMultipleCores <- function( track_memory = FALSE, max_proteins_per_worker = 50L ) { + .checkAFTSolver(aft_solver) if (numberOfCores <= 1L && is.null(BPPARAM)) { return(MSstatsSummarizeWithSingleCore( input, method, impute, censored_symbol, diff --git a/R/dataProcess.R b/R/dataProcess.R index 7fcd6766..dd0f03fe 100755 --- a/R/dataProcess.R +++ b/R/dataProcess.R @@ -158,6 +158,7 @@ dataProcess = function( list(method = summaryMethod, equal_var = equalFeatureVar), list(symbol = censoredInt, MB = MBimpute), colnames(raw)) + .checkAFTSolver(aft_solver) peptides_dict = makePeptidesDictionary(as.data.table(unclass(raw)), normalization) input = MSstatsPrepareForDataProcess(raw, logTrans, fix_missing) @@ -229,7 +230,7 @@ dataProcess = function( MSstatsSummarizeWithSingleCore = function(input, method, impute, censored_symbol, remove50missing, equal_variance, aft_iterations = 90, aft_solver = "cholesky", aft_verbose = FALSE) { - + .checkAFTSolver(aft_solver) is_labeled_reference = "is_labeled_ref" %in% colnames(input) && any(input$is_labeled_ref, na.rm = TRUE) if (is_labeled_reference) { @@ -316,6 +317,7 @@ MSstatsSummarizeSingleLinear = function(single_protein, aft_solver = "cholesky", aft_verbose = FALSE) { ABUNDANCE = RUN = FEATURE = PROTEIN = LogIntensities = NULL + .checkAFTSolver(aft_solver) cols = intersect( colnames(single_protein), @@ -471,6 +473,7 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, aft_verbose = FALSE) { newABUNDANCE = n_obs = n_obs_run = RUN = FEATURE = LABEL = NULL predicted = censored = NULL + .checkAFTSolver(aft_solver) cols = intersect(colnames(single_protein), c("newABUNDANCE", "cen", "RUN", "FEATURE", "ref_covariate")) is_labeled_reference = "is_labeled_ref" %in% colnames(single_protein) && diff --git a/R/utils_checks.R b/R/utils_checks.R index cf994a59..fb34ad99 100644 --- a/R/utils_checks.R +++ b/R/utils_checks.R @@ -79,6 +79,15 @@ MSstatsPrepareForDataProcess = function(input, log_base, fix_missing) { sink() } +#' Check that aft_solver is one of the supported AFT solvers +#' @param aft_solver string: "cholesky", "cg", or "pcg" +#' @keywords internal +#' @noRd +.checkAFTSolver = function(aft_solver) { + checkmate::assertChoice(aft_solver, c("cholesky", "cg", "pcg"), + .var.name = "aft_solver") +} + #' Check validity of parameters to dataProcess function #' @param log_base of logarithmic transformation #' @param normalization_method string: "quantile", "equalizemedians", "FALSE", diff --git a/R/utils_imputation.R b/R/utils_imputation.R index d16bb52a..e9b6b67b 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -589,6 +589,7 @@ #' @noRd .fitAFTModel = function(input, aft_iterations, aft_solver = "cholesky", aft_verbose = FALSE) { + .checkAFTSolver(aft_solver) if (aft_solver == "pcg") { .fitSurvivalCG(input, aft_iterations, use_jacobi_preconditioner = TRUE, verbose = aft_verbose) diff --git a/inst/tinytest/test_dataProcess.R b/inst/tinytest/test_dataProcess.R index 21608d3f..740be683 100644 --- a/inst/tinytest/test_dataProcess.R +++ b/inst/tinytest/test_dataProcess.R @@ -455,3 +455,36 @@ for (solver in setdiff(aft_solvers, "cholesky")) { info = sprintf("MSstatsSummarizeSingleTMP SRM: aft_solver = %s should closely match aft_solver = cholesky", solver) ) } + +srm_input <- make_srm_imputation_input_with_noise(seed = 1) +expect_error( + get_censored_row_predictions(srm_input, "cgp"), + pattern = "aft_solver", + info = "MSstatsSummarizeSingleTMP should reject an unsupported aft_solver" +) +expect_error( + MSstatsSummarizeSingleLinear(srm_input, impute = TRUE, + censored_symbol = "NA", + remove50missing = FALSE, + aft_solver = "cgp"), + pattern = "aft_solver", + info = "MSstatsSummarizeSingleLinear should reject an unsupported aft_solver" +) +expect_error( + MSstatsSummarizeWithSingleCore(srm_input, "TMP", TRUE, "NA", FALSE, TRUE, + aft_solver = "cgp"), + pattern = "aft_solver", + info = "MSstatsSummarizeWithSingleCore should reject an unsupported aft_solver" +) +expect_error( + MSstatsSummarizeWithMultipleCores(srm_input, "TMP", TRUE, "NA", FALSE, + TRUE, aft_solver = "cgp"), + pattern = "aft_solver", + info = "MSstatsSummarizeWithMultipleCores should reject an unsupported aft_solver" +) +expect_error( + dataProcess(DDARawData, aft_solver = "cgp", use_log_file = FALSE, + verbose = FALSE), + pattern = "aft_solver", + info = "dataProcess should reject an unsupported aft_solver before summarization" +) diff --git a/inst/tinytest/test_utils_imputation.R b/inst/tinytest/test_utils_imputation.R index 6a0eda78..2a441d20 100644 --- a/inst/tinytest/test_utils_imputation.R +++ b/inst/tinytest/test_utils_imputation.R @@ -160,6 +160,11 @@ for (aft_solver in c("cg", "pcg")) { "') should attach cg_diagnostics") ) } +expect_error( + MSstats:::.fitAFTModel(noisy_input, number_of_iterations, "cgp"), + pattern = "aft_solver", + info = ".fitAFTModel should reject an unsupported aft_solver instead of falling back to Cholesky" +) expect_silent( MSstats:::.fitSurvivalCG(noisy_input, number_of_iterations, verbose = FALSE) diff --git a/man/MSstatsSummarizeSingleTMP.Rd b/man/MSstatsSummarizeSingleTMP.Rd index 02c5f119..82046bbd 100644 --- a/man/MSstatsSummarizeSingleTMP.Rd +++ b/man/MSstatsSummarizeSingleTMP.Rd @@ -24,6 +24,16 @@ MSstatsSummarizeSingleTMP( \item{remove50missing}{only for method = "TMP"; drops proteins missing >=50\% per peptide in every run} \item{aft_iterations}{number of iterations for AFT model fitting} + +\item{aft_solver}{only used when impute = TRUE; linear solve +used in the AFT imputation model's Newton-Raphson step: "cholesky" +(default, via \code{survival::survreg}), "cg" (conjugate gradient), or +"pcg" (conjugate gradient with a Jacobi/inverse-diagonal preconditioner). +"cg"/"pcg" are experimental.} + +\item{aft_verbose}{if \code{TRUE}, \code{message()} AFT fitting diagnostics +(problem size, elapsed time, and for "cg"/"pcg" per-Newton-iteration +conjugate-gradient counts) for every protein fit. Default \code{FALSE}.} } \value{ list of two data.tables: one with fitted survival model, diff --git a/man/MSstatsSummarizeWithMultipleCores.Rd b/man/MSstatsSummarizeWithMultipleCores.Rd index d2574fde..cc33e43d 100644 --- a/man/MSstatsSummarizeWithMultipleCores.Rd +++ b/man/MSstatsSummarizeWithMultipleCores.Rd @@ -38,6 +38,16 @@ MSstatsSummarizeWithMultipleCores( \item{aft_iterations}{number of AFT model iterations} +\item{aft_solver}{only used when impute = TRUE; linear solve +used in the AFT imputation model's Newton-Raphson step: "cholesky" +(default, via \code{survival::survreg}), "cg" (conjugate gradient), or +"pcg" (conjugate gradient with a Jacobi/inverse-diagonal preconditioner). +"cg"/"pcg" are experimental.} + +\item{aft_verbose}{if \code{TRUE}, \code{message()} AFT fitting diagnostics +(problem size, elapsed time, and for "cg"/"pcg" per-Newton-iteration +conjugate-gradient counts) for every protein fit. Default \code{FALSE}.} + \item{verbose}{whether to print verbose output} \item{BPPARAM}{optional \code{BiocParallelParam} instance} diff --git a/man/MSstatsSummarizeWithSingleCore.Rd b/man/MSstatsSummarizeWithSingleCore.Rd index 94f4ceb3..c004bded 100644 --- a/man/MSstatsSummarizeWithSingleCore.Rd +++ b/man/MSstatsSummarizeWithSingleCore.Rd @@ -30,6 +30,16 @@ MSstatsSummarizeWithSingleCore( \item{equal_variance}{only for method = "linear"; assume equal variance among feature intensities} \item{aft_iterations}{Number of iterations for AFT model fitting. Default is 90.} + +\item{aft_solver}{only used when impute = TRUE; linear solve +used in the AFT imputation model's Newton-Raphson step: "cholesky" +(default, via \code{survival::survreg}), "cg" (conjugate gradient), or +"pcg" (conjugate gradient with a Jacobi/inverse-diagonal preconditioner). +"cg"/"pcg" are experimental.} + +\item{aft_verbose}{if \code{TRUE}, \code{message()} AFT fitting diagnostics +(problem size, elapsed time, and for "cg"/"pcg" per-Newton-iteration +conjugate-gradient counts) for every protein fit. Default \code{FALSE}.} } \value{ list of length one with run-level data. diff --git a/man/reexports.Rd b/man/reexports.Rd index eeac4273..04f47fc4 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -21,6 +21,6 @@ These objects are imported from other packages. Follow the links below to see their documentation. \describe{ - \item{MSstatsConvert}{\code{\link[MSstatsConvert:DIANNtoMSstatsFormat]{DIANNtoMSstatsFormat()}}, \code{\link[MSstatsConvert:DIAUmpiretoMSstatsFormat]{DIAUmpiretoMSstatsFormat()}}, \code{\link[MSstatsConvert:FragPipetoMSstatsFormat]{FragPipetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MZMinetoMSstatsFormat]{MZMinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MaxQtoMSstatsFormat]{MaxQtoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenMStoMSstatsFormat]{OpenMStoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenSWATHtoMSstatsFormat]{OpenSWATHtoMSstatsFormat()}}, \code{\link[MSstatsConvert:PDtoMSstatsFormat]{PDtoMSstatsFormat()}}, \code{\link[MSstatsConvert:ProgenesistoMSstatsFormat]{ProgenesistoMSstatsFormat()}}, \code{\link[MSstatsConvert:SkylinetoMSstatsFormat]{SkylinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:SpectronauttoMSstatsFormat]{SpectronauttoMSstatsFormat()}}} + \item{MSstatsConvert}{\code{\link[MSstatsConvert:DIANNtoMSstatsFormat]{DIANNtoMSstatsFormat()}}, \code{\link[MSstatsConvert:DIAUmpiretoMSstatsFormat]{DIAUmpiretoMSstatsFormat()}}, \code{\link[MSstatsConvert:FragPipetoMSstatsFormat]{FragPipetoMSstatsFormat()}}, \code{\link[MSstatsConvert:MaxQtoMSstatsFormat]{MaxQtoMSstatsFormat()}}, \code{\link[MSstatsConvert:MZMinetoMSstatsFormat]{MZMinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenMStoMSstatsFormat]{OpenMStoMSstatsFormat()}}, \code{\link[MSstatsConvert:OpenSWATHtoMSstatsFormat]{OpenSWATHtoMSstatsFormat()}}, \code{\link[MSstatsConvert:PDtoMSstatsFormat]{PDtoMSstatsFormat()}}, \code{\link[MSstatsConvert:ProgenesistoMSstatsFormat]{ProgenesistoMSstatsFormat()}}, \code{\link[MSstatsConvert:SkylinetoMSstatsFormat]{SkylinetoMSstatsFormat()}}, \code{\link[MSstatsConvert:SpectronauttoMSstatsFormat]{SpectronauttoMSstatsFormat()}}} }} From a987222de4186b91bd136f487390f1a1a413dc41 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 17:31:31 -0400 Subject: [PATCH 18/30] update cgsolve docs --- R/utils_cgsolve.R | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index b1561d34..35669e13 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -11,9 +11,15 @@ #' support at all. #' #' Iteration always starts from the zero vector and stops once the true -#' (unpreconditioned) residual has shrunk to \code{1e-8} of the size of -#' \code{right_hand_side}, so the tolerance means the same thing whether or -#' not \code{use_jacobi_preconditioner} is set. In exact arithmetic, +#' (unpreconditioned) residual is below +#' \code{1e-8 * max(norm(right_hand_side), 1)} - relative for large +#' right-hand sides, absolute (\code{1e-8}) for small ones - so the +#' tolerance means the same thing whether or not +#' \code{use_jacobi_preconditioner} is set. The absolute floor is +#' intentional: the caller (\code{.fitSurvivalCG}) passes a gradient, so a +#' right-hand side this small means the Newton iteration has already +#' converged, and \code{.fitSurvivalCG} judges convergence by the change in +#' log-likelihood rather than by the step returned here. In exact arithmetic, #' conjugate gradient converges within \code{nrow(coefficient_matrix)} #' steps, but rounding error erodes that guarantee as the system grows, so #' up to \code{10 * nrow(coefficient_matrix)} steps are allowed. From 988e74e5908c69848afbf85758df011eeea258e8 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 17:37:46 -0400 Subject: [PATCH 19/30] deal with convergence warning problem --- R/dataProcess.R | 3 ++- inst/tinytest/test_dataProcess.R | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/R/dataProcess.R b/R/dataProcess.R index dd0f03fe..44089b7e 100755 --- a/R/dataProcess.R +++ b/R/dataProcess.R @@ -502,10 +502,11 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, .fitAFTModel(fit_data, aft_iterations, aft_solver, aft_verbose) }, warning = function(w) { warning_message = conditionMessage(w) - if (grepl("converge", warning_message, ignore.case = TRUE)) { + if (grepl("converg", warning_message, ignore.case = TRUE)) { convergence_messages <<- c(convergence_messages, warning_message) converged <<- FALSE + invokeRestart("muffleWarning") } }) diff --git a/inst/tinytest/test_dataProcess.R b/inst/tinytest/test_dataProcess.R index 740be683..fcb46b88 100644 --- a/inst/tinytest/test_dataProcess.R +++ b/inst/tinytest/test_dataProcess.R @@ -457,6 +457,42 @@ for (solver in setdiff(aft_solvers, "cholesky")) { } srm_input <- make_srm_imputation_input_with_noise(seed = 1) + +# A non-converged AFT fit should be reported once, as the combined +# per-protein message, not additionally as the solver's raw warning, and +# must not be used for imputation. aft_iterations = 2 is the smallest +# budget that leaves every solver unconverged on this input (survreg +# does not warn at maxiter = 1 here). +for (solver in aft_solvers) { + raw_warnings <- character(0) + convergence_messages <- character(0) + unconverged_summary <- withCallingHandlers( + MSstatsSummarizeSingleTMP(srm_input, impute = TRUE, + censored_symbol = "NA", + remove50missing = FALSE, + aft_iterations = 2, aft_solver = solver), + warning = function(w) { + raw_warnings <<- c(raw_warnings, conditionMessage(w)) + invokeRestart("muffleWarning") + }, + message = function(m) { + convergence_messages <<- c(convergence_messages, + conditionMessage(m)) + invokeRestart("muffleMessage") + }) + expect_equal( + sum(grepl("CONVERGENCE WARNING", convergence_messages)), 1L, + info = sprintf("MSstatsSummarizeSingleTMP (aft_solver = %s): a non-converged fit should emit one combined convergence message", solver) + ) + expect_false( + any(grepl("converge", raw_warnings, ignore.case = TRUE)), + info = sprintf("MSstatsSummarizeSingleTMP (aft_solver = %s): the solver's raw convergence warning should be muffled", solver) + ) + expect_true( + all(is.na(unconverged_summary[[2]]$predicted)), + info = sprintf("MSstatsSummarizeSingleTMP (aft_solver = %s): a non-converged fit should not be used for imputation", solver) + ) +} expect_error( get_censored_row_predictions(srm_input, "cgp"), pattern = "aft_solver", From 78f37c3f0ca7bcaf1e53110f6433bdaa18d570a5 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 17:43:47 -0400 Subject: [PATCH 20/30] adjust tests for warning messages aft imputation --- R/dataProcess.R | 2 +- inst/tinytest/test_dataProcess.R | 66 +++++++++++++++----------------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/R/dataProcess.R b/R/dataProcess.R index 44089b7e..d9f9c1bf 100755 --- a/R/dataProcess.R +++ b/R/dataProcess.R @@ -502,7 +502,7 @@ MSstatsSummarizeSingleTMP = function(single_protein, impute, censored_symbol, .fitAFTModel(fit_data, aft_iterations, aft_solver, aft_verbose) }, warning = function(w) { warning_message = conditionMessage(w) - if (grepl("converg", warning_message, ignore.case = TRUE)) { + if (grepl("converge", warning_message, ignore.case = TRUE)) { convergence_messages <<- c(convergence_messages, warning_message) converged <<- FALSE diff --git a/inst/tinytest/test_dataProcess.R b/inst/tinytest/test_dataProcess.R index fcb46b88..e5a6f5dd 100644 --- a/inst/tinytest/test_dataProcess.R +++ b/inst/tinytest/test_dataProcess.R @@ -458,41 +458,6 @@ for (solver in setdiff(aft_solvers, "cholesky")) { srm_input <- make_srm_imputation_input_with_noise(seed = 1) -# A non-converged AFT fit should be reported once, as the combined -# per-protein message, not additionally as the solver's raw warning, and -# must not be used for imputation. aft_iterations = 2 is the smallest -# budget that leaves every solver unconverged on this input (survreg -# does not warn at maxiter = 1 here). -for (solver in aft_solvers) { - raw_warnings <- character(0) - convergence_messages <- character(0) - unconverged_summary <- withCallingHandlers( - MSstatsSummarizeSingleTMP(srm_input, impute = TRUE, - censored_symbol = "NA", - remove50missing = FALSE, - aft_iterations = 2, aft_solver = solver), - warning = function(w) { - raw_warnings <<- c(raw_warnings, conditionMessage(w)) - invokeRestart("muffleWarning") - }, - message = function(m) { - convergence_messages <<- c(convergence_messages, - conditionMessage(m)) - invokeRestart("muffleMessage") - }) - expect_equal( - sum(grepl("CONVERGENCE WARNING", convergence_messages)), 1L, - info = sprintf("MSstatsSummarizeSingleTMP (aft_solver = %s): a non-converged fit should emit one combined convergence message", solver) - ) - expect_false( - any(grepl("converge", raw_warnings, ignore.case = TRUE)), - info = sprintf("MSstatsSummarizeSingleTMP (aft_solver = %s): the solver's raw convergence warning should be muffled", solver) - ) - expect_true( - all(is.na(unconverged_summary[[2]]$predicted)), - info = sprintf("MSstatsSummarizeSingleTMP (aft_solver = %s): a non-converged fit should not be used for imputation", solver) - ) -} expect_error( get_censored_row_predictions(srm_input, "cgp"), pattern = "aft_solver", @@ -524,3 +489,34 @@ expect_error( pattern = "aft_solver", info = "dataProcess should reject an unsupported aft_solver before summarization" ) + +for (solver in aft_solvers) { + raw_warnings <- character(0) + convergence_messages <- character(0) + unconverged_summary <- withCallingHandlers( + MSstatsSummarizeSingleTMP(srm_input, impute = TRUE, + censored_symbol = "NA", + remove50missing = FALSE, + aft_iterations = 2, aft_solver = solver), + warning = function(w) { + raw_warnings <<- c(raw_warnings, conditionMessage(w)) + invokeRestart("muffleWarning") + }, + message = function(m) { + convergence_messages <<- c(convergence_messages, + conditionMessage(m)) + invokeRestart("muffleMessage") + }) + expect_equal( + sum(grepl("CONVERGENCE WARNING", convergence_messages)), 1L, + info = sprintf("MSstatsSummarizeSingleTMP (aft_solver = %s): a non-converged fit should emit one combined convergence message", solver) + ) + expect_false( + any(grepl("converge", raw_warnings, ignore.case = TRUE)), + info = sprintf("MSstatsSummarizeSingleTMP (aft_solver = %s): the solver's raw convergence warning should be muffled", solver) + ) + expect_true( + all(is.na(unconverged_summary[[2]]$predicted)), + info = sprintf("MSstatsSummarizeSingleTMP (aft_solver = %s): a non-converged fit should not be used for imputation", solver) + ) +} From f1c2f14f003aa33249fb0f31eea8370fc7dc6791 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 17:50:16 -0400 Subject: [PATCH 21/30] remove unnecessary comments --- R/utils_cgsolve.R | 14 ----------- R/utils_imputation.R | 57 -------------------------------------------- 2 files changed, 71 deletions(-) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index 35669e13..38b99d46 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -63,10 +63,6 @@ identity } - # The residual measures how far the current guess is from solving the - # system. Conjugate gradient starts out searching in the - # preconditioner-adjusted residual direction (with no preconditioner, - # this is just the residual itself). residual = right_hand_side - drop(coefficient_matrix %*% solution) preconditioned_residual = apply_preconditioner(residual) search_direction = preconditioned_residual @@ -74,9 +70,6 @@ residual_dot_preconditioned_residual = sum(residual * preconditioned_residual) - # Stop once the residual has shrunk far enough, relative to the size of - # the right-hand side (falling back to an absolute scale when that size - # is tiny). convergence_threshold = (relative_tolerance * max(sqrt(sum(right_hand_side^2)), 1))^2 @@ -88,9 +81,6 @@ break } iterations_used = iteration - - # How far moving along the search direction changes things, as - # measured through the matrix itself. matrix_times_search_direction = drop(coefficient_matrix %*% search_direction) curvature = sum(search_direction * matrix_times_search_direction) @@ -102,15 +92,11 @@ break } - # Move as far as possible along the search direction without - # overshooting the solution, then see how much residual remains. step_length = residual_dot_preconditioned_residual / curvature solution = solution + step_length * search_direction residual = residual - step_length * matrix_times_search_direction new_residual_size = sum(residual * residual) - # Choose the next search direction so it doesn't undo the progress - # made by earlier directions. new_preconditioned_residual = apply_preconditioner(residual) new_residual_dot_preconditioned_residual = sum(residual * new_preconditioned_residual) diff --git a/R/utils_imputation.R b/R/utils_imputation.R index e9b6b67b..2c4d0c99 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -31,8 +31,6 @@ n_runs = data.table::uniqueN(input[missingness_filter, RUN]) is_labeled = data.table::uniqueN(input$LABEL) > 1 - # With too few uncensored observations, there isn't enough information - # left to also estimate a separate effect per feature. not_enough_data_for_feature_effect = n_total < n_features + n_runs - 1 if (is_labeled) { @@ -68,12 +66,9 @@ #' @keywords internal #' @noRd .fitSurvival = function(input, aft_iterations, verbose = FALSE) { - # TODO: set.seed here? set.seed(100) aft_formula = .buildAFTFormula(input) if (verbose) { - # survreg builds these internally; rebuilding them here is only - # worth the extra work when the counts are actually reported. model_frame = model.frame(aft_formula, data = input) design_matrix = model.matrix(attr(model_frame, "terms"), model_frame) message(sprintf( @@ -137,9 +132,6 @@ observed_value, exact_indicator) { scale = exp(log_scale) inverse_scale_squared = 1 / scale^2 - - # How far the observation sits from its predicted value, in raw units - # and in standard deviations. distance_from_prediction = observed_value - linear_predictor standardized_distance = distance_from_prediction / scale @@ -148,10 +140,6 @@ pnorm(standardized_distance) is_exact_observation = (exact_indicator == 1) - # --- exact (uncensored) observations -------------------------------- - # log-likelihood contribution is log(density) - log(scale); what - # follows is that expression's derivatives wrt linear_predictor and - # log_scale. exact_log_likelihood = log(density_at_standardized_distance) - log_scale exact_gradient_wrt_linear_predictor = standardized_distance / scale @@ -173,11 +161,6 @@ exact_gradient_wrt_log_scale = exact_gradient_wrt_log_scale_before_adjustment - 1 - # Guard against the density underflowing to exactly zero (only - # happens for astronomically large |standardized_distance|, e.g. from - # a wild early Newton guess). Any reasonable derivative works here, - # since the collapsed log-likelihood itself is what triggers - # step-halving. exact_density_underflowed = density_at_standardized_distance <= 0 exact_log_likelihood = ifelse(exact_density_underflowed, -200, exact_log_likelihood) @@ -195,10 +178,6 @@ exact_density_underflowed, 0, exact_second_derivative_wrt_log_scale) - # --- left-censored observations (true value <= the recorded ceiling) - - # log-likelihood contribution is log(Phi(standardized_distance)); - # "censoring_hazard" plays the same role for these rows that the - # density itself plays above. censored_log_likelihood = log(cumulative_probability_at_standardized_distance) censoring_hazard = density_at_standardized_distance / @@ -221,9 +200,6 @@ distance_from_prediction^2 * censored_log_density_curvature - censored_gradient_wrt_log_scale * (1 + censored_gradient_wrt_log_scale) - # Same underflow guard as above, triggered when the cumulative - # probability collapses to zero (standardized_distance very - # negative). censored_probability_underflowed = cumulative_probability_at_standardized_distance <= 0 censored_log_likelihood = ifelse( @@ -335,12 +311,6 @@ observed_value = response[, 1] exact_indicator = response[, 2] - # Initial guess: an ordinary least-squares fit for the regression - # coefficients (treating the detection-limit ceiling already - # substituted into censored rows as if it were observed), and the - # residual standard deviation for the scale parameter. A - # rank-deficient design leaves some coefficients unidentified - # (reported as NA by lm.fit); start those at zero. initial_fit = lm.fit(design_matrix, observed_value) coefficients = initial_fit$coefficients coefficients[!is.finite(coefficients)] = 0 @@ -361,8 +331,6 @@ } build_information_matrix = function(derivatives) { - # Regression block: -t(X) %*% diag(second_derivative) %*% X, - # computed without forming the diagonal matrix explicitly. regression_block = -crossprod( design_matrix, design_matrix * derivatives$second_derivative_wrt_linear_predictor) @@ -381,13 +349,6 @@ all(is.finite(derivatives$second_derivative_wrt_log_scale)) } - # A Newton step away from the optimum, the exact information matrix - # is not guaranteed to be positive definite. survival::survreg falls - # back, in that situation, to the sum of the outer products of each - # observation's own contribution to the gradient - always positive - # semi-definite by construction, and equal to the exact information - # matrix in expectation (this is the classic Gauss-Newton / BHHH - # approximation). Mirror that fallback here. build_gauss_newton_approximation = function(derivatives) { per_observation_gradient_contributions = cbind( design_matrix * derivatives$gradient_wrt_linear_predictor, @@ -395,11 +356,6 @@ crossprod(per_observation_gradient_contributions) } - # A "not positive definite" result is expected, handled control flow - # here (the Gauss-Newton fallback below exists for exactly that case), - # so its warning is muffled; a genuine "did not converge within - # max_iterations" is not expected/handled, so that warning still - # propagates normally. cg_solve_muffling_pd_warning = function(...) { withCallingHandlers( .cgSolve(...), @@ -410,11 +366,6 @@ }) } - # Returns the Newton step, plus how much conjugate-gradient work it - # took to get there - primary_iterations/fallback_iterations and - # used_fallback are the numbers verbose logging (below) reports, so a - # caller can see how solver choice and problem size trade off against - # iteration count. solve_newton_step = function(information_matrix, derivatives, gradient) { primary_solve = cg_solve_muffling_pd_warning( information_matrix, gradient, @@ -476,11 +427,6 @@ candidate_log_scale = log_scale + newton_step$step[number_of_coefficients + 1] - # Step-halving: if the Newton step overshoots (a non-finite or - # decreasing log-likelihood), back the trial point off toward the - # last accepted one, mirroring survival::survreg's own recovery - # strategy (survreg6.c) rather than simply rejecting the step - # outright. number_of_halvings = 0 repeat { candidate_fit = evaluate_log_likelihood_and_derivatives( @@ -494,9 +440,6 @@ number_of_halvings = number_of_halvings + 1 if (number_of_halvings == 1 && (log_scale - candidate_log_scale) > 1.1) { - # a single huge drop in scale is the most common cause of - # a bad trial; keep the first back-off from cutting scale - # by more than a factor of exp(1.1), same as survreg6.c candidate_log_scale = log_scale - 1.1 } candidate_coefficients = From 2cd10ad91b6d91e56f484e1a9bea1132317c674d Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 18:04:10 -0400 Subject: [PATCH 22/30] fix unit test on convergence --- R/utils_imputation.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 2c4d0c99..8ca21761 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -469,8 +469,8 @@ } if (!converged) { - warning("AFT model (CG solver) used its full iteration budget ", - "without converging; returning the last accepted ", + warning("AFT model (CG solver) did not converge within its ", + "iteration budget; returning the last accepted ", "coefficients") } From 89f07ee8135149167755bd2c799af443f427a940 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 18:55:02 -0400 Subject: [PATCH 23/30] move internal functions outside of .fitSurvivalCG --- R/utils_imputation.R | 247 +++++++++++++++++++++++++++++-------------- 1 file changed, 170 insertions(+), 77 deletions(-) diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 8ca21761..37094b64 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -241,6 +241,162 @@ ) } +#' Evaluate the Gaussian AFT log-likelihood and derivatives at a +#' parameter guess +#' +#' Thin wrapper around \code{.aftGaussianDerivatives} that forms the +#' linear predictor from \code{design_matrix} and \code{coefficients}. +#' +#' @param design_matrix model matrix of the AFT fit. +#' @param coefficients current regression coefficients. +#' @param log_scale current log of the scale parameter. +#' @param observed_value observed (or censoring-threshold) values. +#' @param exact_indicator \code{1} for exact rows, \code{0} for +#' left-censored rows. +#' +#' @return the list returned by \code{.aftGaussianDerivatives}. +#' +#' @keywords internal +#' @noRd +.evaluateAFTLogLikelihood = function(design_matrix, coefficients, + log_scale, observed_value, + exact_indicator) { + .aftGaussianDerivatives( + drop(design_matrix %*% coefficients), log_scale, + observed_value, exact_indicator) +} + +#' Assemble the AFT score vector +#' +#' @param design_matrix model matrix of the AFT fit. +#' @param derivatives output of \code{.aftGaussianDerivatives}. +#' +#' @return gradient of the log-likelihood with respect to the regression +#' coefficients followed by the log scale. +#' +#' @keywords internal +#' @noRd +.buildAFTGradient = function(design_matrix, derivatives) { + c(as.vector(crossprod( + design_matrix, derivatives$gradient_wrt_linear_predictor)), + sum(derivatives$gradient_wrt_log_scale)) +} + +#' Assemble the AFT observed information matrix +#' +#' @param design_matrix model matrix of the AFT fit. +#' @param derivatives output of \code{.aftGaussianDerivatives}. +#' +#' @return negative Hessian of the log-likelihood over the regression +#' coefficients and the log scale. +#' +#' @keywords internal +#' @noRd +.buildAFTInformationMatrix = function(design_matrix, derivatives) { + regression_block = -crossprod( + design_matrix, + design_matrix * derivatives$second_derivative_wrt_linear_predictor) + cross_block = -as.vector( + crossprod(design_matrix, derivatives$cross_derivative)) + scale_block = -sum(derivatives$second_derivative_wrt_log_scale) + rbind(cbind(regression_block, cross_block), + c(cross_block, scale_block)) +} + +#' Check that an AFT log-likelihood evaluation is finite +#' +#' @param derivatives output of \code{.aftGaussianDerivatives}. +#' +#' @return \code{TRUE} if the log-likelihood and all first/second +#' derivatives used by the Newton step are finite. +#' +#' @keywords internal +#' @noRd +.isFiniteAFTFit = function(derivatives) { + is.finite(derivatives$log_likelihood) && + all(is.finite(derivatives$gradient_wrt_linear_predictor)) && + all(is.finite(derivatives$gradient_wrt_log_scale)) && + all(is.finite(derivatives$second_derivative_wrt_linear_predictor)) && + all(is.finite(derivatives$second_derivative_wrt_log_scale)) +} + +#' Gauss-Newton (outer-product-of-gradients) approximation to the AFT +#' information matrix +#' +#' Always positive semi-definite, so it is used as a fallback when the +#' observed information matrix is not positive definite. +#' +#' @param design_matrix model matrix of the AFT fit. +#' @param derivatives output of \code{.aftGaussianDerivatives}. +#' +#' @return crossproduct of the per-observation gradient contributions. +#' +#' @keywords internal +#' @noRd +.buildGaussNewtonApproximation = function(design_matrix, derivatives) { + per_observation_gradient_contributions = cbind( + design_matrix * derivatives$gradient_wrt_linear_predictor, + derivatives$gradient_wrt_log_scale) + crossprod(per_observation_gradient_contributions) +} + +#' Run \code{.cgSolve} with its "not positive definite" warning muffled +#' +#' @param ... passed to \code{.cgSolve}. +#' +#' @return the output of \code{.cgSolve}. +#' +#' @keywords internal +#' @noRd +.cgSolveMufflingPDWarning = function(...) { + withCallingHandlers( + .cgSolve(...), + warning = function(w) { + if (grepl("not positive definite", conditionMessage(w))) { + invokeRestart("muffleWarning") + } + }) +} + +#' Solve for one AFT Newton-Raphson step with conjugate gradient +#' +#' Solves \code{information_matrix \%*\% step = gradient}; if the +#' information matrix turns out not to be positive definite, re-solves +#' against the Gauss-Newton approximation instead. +#' +#' @param design_matrix model matrix of the AFT fit. +#' @param information_matrix output of \code{.buildAFTInformationMatrix}. +#' @param derivatives output of \code{.aftGaussianDerivatives}. +#' @param gradient output of \code{.buildAFTGradient}. +#' @param use_jacobi_preconditioner passed to \code{.cgSolve}. +#' +#' @return a list with the \code{step}, \code{primary_iterations}, +#' \code{used_fallback}, and \code{fallback_iterations}. +#' +#' @keywords internal +#' @noRd +.solveAFTNewtonStep = function(design_matrix, information_matrix, + derivatives, gradient, + use_jacobi_preconditioner) { + primary_solve = .cgSolveMufflingPDWarning( + information_matrix, gradient, + use_jacobi_preconditioner = use_jacobi_preconditioner) + if (primary_solve$positive_definite) { + list(step = primary_solve$solution, + primary_iterations = primary_solve$iterations, + used_fallback = FALSE, fallback_iterations = 0L) + } else { + fallback_solve = .cgSolve( + .buildGaussNewtonApproximation(design_matrix, derivatives), + gradient, + use_jacobi_preconditioner = use_jacobi_preconditioner) + list(step = fallback_solve$solution, + primary_iterations = primary_solve$iterations, + used_fallback = TRUE, + fallback_iterations = fallback_solve$iterations) + } +} + #' Fit a Gaussian, left-censored AFT model with a conjugate-gradient #' Newton step #' @@ -317,76 +473,9 @@ residual_standard_deviation = sd(initial_fit$residuals) log_scale = log(max(residual_standard_deviation, 1e-4)) - evaluate_log_likelihood_and_derivatives = function(coefficients, - log_scale) { - .aftGaussianDerivatives( - drop(design_matrix %*% coefficients), log_scale, - observed_value, exact_indicator) - } - - build_gradient = function(derivatives) { - c(as.vector(crossprod( - design_matrix, derivatives$gradient_wrt_linear_predictor)), - sum(derivatives$gradient_wrt_log_scale)) - } - - build_information_matrix = function(derivatives) { - regression_block = -crossprod( - design_matrix, - design_matrix * derivatives$second_derivative_wrt_linear_predictor) - cross_block = -as.vector( - crossprod(design_matrix, derivatives$cross_derivative)) - scale_block = -sum(derivatives$second_derivative_wrt_log_scale) - rbind(cbind(regression_block, cross_block), - c(cross_block, scale_block)) - } - - is_finite_fit = function(derivatives) { - is.finite(derivatives$log_likelihood) && - all(is.finite(derivatives$gradient_wrt_linear_predictor)) && - all(is.finite(derivatives$gradient_wrt_log_scale)) && - all(is.finite(derivatives$second_derivative_wrt_linear_predictor)) && - all(is.finite(derivatives$second_derivative_wrt_log_scale)) - } - - build_gauss_newton_approximation = function(derivatives) { - per_observation_gradient_contributions = cbind( - design_matrix * derivatives$gradient_wrt_linear_predictor, - derivatives$gradient_wrt_log_scale) - crossprod(per_observation_gradient_contributions) - } - - cg_solve_muffling_pd_warning = function(...) { - withCallingHandlers( - .cgSolve(...), - warning = function(w) { - if (grepl("not positive definite", conditionMessage(w))) { - invokeRestart("muffleWarning") - } - }) - } - - solve_newton_step = function(information_matrix, derivatives, gradient) { - primary_solve = cg_solve_muffling_pd_warning( - information_matrix, gradient, - use_jacobi_preconditioner = use_jacobi_preconditioner) - if (primary_solve$positive_definite) { - list(step = primary_solve$solution, - primary_iterations = primary_solve$iterations, - used_fallback = FALSE, fallback_iterations = 0L) - } else { - fallback_solve = .cgSolve( - build_gauss_newton_approximation(derivatives), gradient, - use_jacobi_preconditioner = use_jacobi_preconditioner) - list(step = fallback_solve$solution, - primary_iterations = primary_solve$iterations, - used_fallback = TRUE, - fallback_iterations = fallback_solve$iterations) - } - } - current_fit = - evaluate_log_likelihood_and_derivatives(coefficients, log_scale) + .evaluateAFTLogLikelihood(design_matrix, coefficients, log_scale, + observed_value, exact_indicator) current_log_likelihood = current_fit$log_likelihood number_of_iterations_used = 0 converged = FALSE @@ -400,10 +489,12 @@ number_of_iterations_used = iteration iteration_start_time = Sys.time() - gradient = build_gradient(current_fit) - information_matrix = build_information_matrix(current_fit) - newton_step = - solve_newton_step(information_matrix, current_fit, gradient) + gradient = .buildAFTGradient(design_matrix, current_fit) + information_matrix = + .buildAFTInformationMatrix(design_matrix, current_fit) + newton_step = .solveAFTNewtonStep( + design_matrix, information_matrix, current_fit, gradient, + use_jacobi_preconditioner) elapsed_seconds = as.numeric(Sys.time() - iteration_start_time, units = "secs") @@ -429,9 +520,10 @@ number_of_halvings = 0 repeat { - candidate_fit = evaluate_log_likelihood_and_derivatives( - candidate_coefficients, candidate_log_scale) - candidate_improves = is_finite_fit(candidate_fit) && + candidate_fit = .evaluateAFTLogLikelihood( + design_matrix, candidate_coefficients, candidate_log_scale, + observed_value, exact_indicator) + candidate_improves = .isFiniteAFTFit(candidate_fit) && candidate_fit$log_likelihood >= current_log_likelihood if (candidate_improves || iterations_remaining <= 0) { break @@ -485,7 +577,8 @@ sum(cg_diagnostics$elapsed_seconds), converged)) } - final_information_matrix = build_information_matrix(current_fit) + final_information_matrix = + .buildAFTInformationMatrix(design_matrix, current_fit) variance_covariance_matrix = tryCatch( solve(final_information_matrix), error = function(e) MASS::ginv(final_information_matrix)) From 7225f7836e79ccc500f4ae823694cf9a3af01b88 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 19:11:55 -0400 Subject: [PATCH 24/30] update documentation on the math claims --- R/utils_cgsolve.R | 209 +++++++++++++++++++++++++++++++++++++++++++ R/utils_imputation.R | 93 +++++++++++++++++++ 2 files changed, 302 insertions(+) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index 38b99d46..8b9ade64 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -24,6 +24,215 @@ #' steps, but rounding error erodes that guarantee as the system grows, so #' up to \code{10 * nrow(coefficient_matrix)} steps are allowed. #' +#' @section The math behind conjugate gradient: +#' Notation (with the variable each symbol lives in): \code{A} is +#' \code{coefficient_matrix} (n x n, symmetric positive definite), \code{b} +#' is \code{right_hand_side}, \code{x_k} is \code{solution} after k steps, +#' \code{r_k = b - A x_k} is \code{residual}, \code{M^-1} is the +#' preconditioner (\code{diag(A)^-1} for Jacobi, the identity otherwise), +#' \code{z_k = M^-1 r_k} is \code{preconditioned_residual}, and \code{p_k} +#' is \code{search_direction}. A prime (') denotes transpose, so +#' \code{u'v} is a dot product, computed here as \code{sum(u * v)}. +#' +#' \strong{1. Solving Ax = b as a minimization.} Because \code{A} is +#' symmetric positive definite, the quadratic +#' \preformatted{ +#' phi(x) = (1/2) x'A x - b'x +#' } +#' is a bowl with a unique minimum. Its gradient is +#' \code{grad phi(x) = A x - b = -r}, so setting the derivative to zero +#' gives exactly \code{A x = b}: the minimizer of \code{phi} is the +#' solution, and the residual is the negative gradient (the steepest +#' downhill direction). +#' +#' \strong{2. One iteration, written as matrix operations.} Starting from +#' \code{x_0 = 0}, \code{r_0 = b}, \code{z_0 = M^-1 r_0}, +#' \code{p_0 = z_0}, each step computes +#' \preformatted{ +#' q_k = A p_k (the only matrix-vector +#' product per iteration) +#' alpha_k = (r_k' z_k) / (p_k' q_k) step length; p_k' A p_k is +#' the "curvature" +#' x_k+1 = x_k + alpha_k p_k +#' r_k+1 = r_k - alpha_k q_k (= b - A x_k+1, updated +#' without a new product) +#' z_k+1 = M^-1 r_k+1 (elementwise scaling for +#' Jacobi) +#' beta_k = (r_k+1' z_k+1) / (r_k' z_k) +#' p_k+1 = z_k+1 + beta_k p_k +#' } +#' In the code, \code{q_k} is \code{matrix_times_search_direction}, +#' \code{p_k' q_k} is \code{curvature}, \code{alpha_k} is +#' \code{step_length}, and \code{r_k' z_k} is +#' \code{residual_dot_preconditioned_residual}. The per-step cost is one +#' n x n matrix-vector product (O(n^2)) plus a handful of O(n) dot products +#' and vector updates, and only the current \code{x}, \code{r}, \code{z}, +#' and \code{p} are kept - no earlier directions need to be stored. +#' +#' \strong{3. Where alpha comes from: residuals orthogonal to the step +#' taken.} Given a direction \code{p_k}, pick the step that minimizes +#' \code{phi} along that line by setting the derivative to zero: +#' \preformatted{ +#' d/d alpha phi(x_k + alpha p_k) +#' = p_k' (A x_k + alpha A p_k - b) +#' = -p_k' r_k + alpha p_k' A p_k = 0 +#' => alpha_k = (p_k' r_k) / (p_k' A p_k) +#' } +#' The zero-derivative condition is the same statement as +#' \code{p_k' r_k+1 = 0}: the new residual (the new downhill direction) is +#' orthogonal to the direction just searched, so there is nothing left to +#' gain along \code{p_k}. Since \code{p_k = z_k + beta_k-1 p_k-1} and +#' \code{p_k-1' r_k = 0} by the same argument one step earlier, +#' \code{p_k' r_k = z_k' r_k}, which is the numerator used in the code. +#' +#' \strong{4. Where beta comes from: A-conjugate directions.} Steepest +#' descent (always stepping along \code{r_k}) also does an exact line +#' search, but a later step can undo progress made along an earlier +#' direction, so it zig-zags across narrow valleys. Conjugate gradient +#' avoids this by choosing each new direction to be \emph{A-conjugate} to +#' the previous one: +#' \preformatted{ +#' p_k+1' A p_k = 0 +#' => (z_k+1 + beta_k p_k)' A p_k = 0 +#' => beta_k = -(z_k+1' A p_k) / (p_k' A p_k) +#' } +#' Substituting \code{A p_k = (r_k - r_k+1) / alpha_k} (from the residual +#' update) together with the orthogonality \code{z_k' r_k+1 = 0} collapses +#' this to \code{beta_k = (r_k+1' z_k+1) / (r_k' z_k)}, a ratio of two +#' dot products that are already on hand. \code{p_i' A p_j = 0} says the +#' directions are orthogonal under the inner product +#' \code{_A = u' A v} - i.e. they would be ordinary perpendicular +#' vectors if space were stretched by \code{A} so the elliptical contours +#' of \code{phi} became circles. By induction the short recurrence gives +#' this for all pairs, not just neighbours: +#' \preformatted{ +#' p_i' A p_j = 0 for all i != j (directions A-conjugate) +#' r_i' M^-1 r_j = 0 for all i != j (residuals orthogonal; plain +#' r_i' r_j = 0 when M = I) +#' } +#' +#' \strong{5. Verifying optimality with derivatives, and why this is +#' fast.} After k steps \code{x_k = alpha_0 p_0 + ... + alpha_k-1 p_k-1}. +#' Take the derivative of \code{phi} with respect to the coefficient on +#' any earlier direction \code{p_j} (j < k): +#' \preformatted{ +#' d phi / d c_j = p_j' (A x_k - b) = -p_j' r_k = 0 +#' } +#' Every one of these partial derivatives is zero, so \code{x_k} is not +#' just the best point on the latest line but the exact minimizer of +#' \code{phi} over the whole subspace \code{span(p_0, ..., p_k-1)} - which +#' equals the Krylov subspace \code{span(r_0, A r_0, ..., A^(k-1) r_0)} +#' (with \code{M^-1} folded in when preconditioned). Conjugacy is what +#' makes this possible: writing the exact answer as +#' \code{x* = sum_j alpha_j p_j} and multiplying \code{A x* = b} on the +#' left by \code{p_j'} kills every cross term, leaving +#' \code{alpha_j = (p_j' b) / (p_j' A p_j)} - each coefficient is +#' determined independently, so each direction is solved once and never +#' revisited. With n independent directions available, exact arithmetic +#' reaches the solution in at most n steps. +#' +#' \strong{6. Why clustered eigenvalues make it faster still.} Because +#' \code{x_k} is optimal over the Krylov subspace, its error +#' \code{e_k = x* - x_k} can be written \code{e_k = P_k(A) e_0} for the +#' degree-k polynomial \code{P_k} with \code{P_k(0) = 1} that minimizes the +#' A-norm of the error. Expanding \code{e_0 = sum_i c_i v_i} in the +#' eigenvectors \code{v_i} of \code{A} (eigenvalues \code{lambda_i}): +#' \preformatted{ +#' ||e_k||_A^2 = sum_i lambda_i c_i^2 P_k(lambda_i)^2 +#' ||e_k||_A / ||e_0||_A <= min over P_k of max_i |P_k(lambda_i)| +#' } +#' so convergence depends on how many \emph{distinct groups} of +#' eigenvalues there are, not on n. If the eigenvalues fall into m tight +#' clusters, a degree-m polynomial with one root near the centre of each +#' cluster is small at every eigenvalue, and CG is essentially done after +#' about m steps. Intuitively, within a cluster \code{A} acts almost like a +#' single scalar, so the curvature \code{p' A p / p' p} - and hence the +#' step length \code{alpha} - is nearly the same for every eigen-direction +#' in that cluster; one step of that size removes the error along all of +#' those directions at once instead of one at a time (in the extreme case +#' \code{A = c I}, a single step solves the system exactly). When the +#' spectrum is spread out instead, the standard bound +#' \preformatted{ +#' ||e_k||_A <= 2 ((sqrt(kappa) - 1) / (sqrt(kappa) + 1))^k ||e_0||_A, +#' kappa = lambda_max / lambda_min +#' } +#' applies. This is the motivation for the Jacobi preconditioner: CG is +#' effectively run on \code{M^-1 A}, and for a diagonally dominant AFT +#' information matrix dividing by the diagonal pulls the eigenvalues +#' toward 1 - clustering them and shrinking \code{kappa}. +#' +#' Finally, the curvature \code{p_k' A p_k} must be positive for +#' \code{phi} to have a minimum along \code{p_k}; if it is zero, negative, +#' or non-finite, \code{A} is not positive definite along that direction, +#' the line search has no minimizer, and the loop stops and reports +#' \code{positive_definite = FALSE}. +#' +#' @section Computational cost: +#' \strong{Cost per solve is k matrix-vector products.} Each iteration +#' does exactly one product \code{A p} plus a fixed number of length-n dot +#' products and vector updates (O(n)). Over k iterations the total is +#' \preformatted{ +#' O(k * (cost of one A p + n)) +#' } +#' Nothing here factorizes \code{A} (a Cholesky factorization costs +#' O(n^3)) or builds any matrix; \code{A} is only ever multiplied by a +#' vector. When a product \code{A p} costs O(n) - proportional to the +#' number of nonzero entries of \code{A} - a full solve costs O(k * n): +#' linear in the problem size for a fixed number of iterations. +#' +#' \strong{k stays small on MSstats data.} Section 6 above shows k is +#' governed by the number of eigenvalue clusters, not by n. The AFT +#' information matrix is dominated by its diagonal (each parameter's own +#' curvature is much larger than its cross-terms), so Jacobi +#' preconditioning pulls most of the spectrum of \code{M^-1 A} to 1, with +#' only a few outlying eigenvalues. For example, in a simulated +#' 12-feature x 10-run protein (n = 22 unknowns, 20\% censored), 14 of the +#' 22 preconditioned eigenvalues lie in [0.9, 1.1], the condition number +#' drops from about 234 to about 103, and CG reaches tolerance in 12 +#' iterations instead of 16. Because the bulk cluster is handled in a few +#' steps, and the remaining iterations are spent on the few outliers, k +#' grows much more slowly than n. +#' +#' \strong{Symmetry and sparsity: only the upper-right block carries real +#' work.} For the \code{~ FEATURE + RUN} model, every row of the design +#' matrix \code{X} has exactly one feature indicator and one run indicator. +#' In \code{A = -X' W X} (plus the log-scale row and column), this means: +#' \itemize{ +#' \item the feature-feature block is \emph{diagonal} - two different +#' features never appear in the same row, so their cross-term is zero; +#' \item the run-run block is likewise \emph{diagonal}; +#' \item the only dense off-diagonal content is the feature x run block +#' \code{C} in the upper-right corner (one entry per feature/run cell +#' that has data), together with the intercept and log-scale border +#' rows; +#' \item by symmetry the lower-left block is \code{C'}, so it carries no +#' new information. +#' } +#' Ignoring the border, the product therefore reduces to +#' \preformatted{ +#' A = [ D_F C ] A p = [ D_F p_F + C p_R ] +#' [ C' D_R ] [ C' p_F + D_R p_R ] +#' } +#' with \code{D_F} and \code{D_R} diagonal: two elementwise scalings plus +#' one pass over \code{C}, used once as-is and once transposed. The +#' nonzero count is about \code{F + R + 2 * (number of feature/run cells)}, +#' i.e. proportional to the number of observations, rather than the +#' \code{n^2} entries of a dense matrix. Equivalently, +#' \code{A p = -X' (w * (X p))} can be computed without forming \code{A} +#' at all, in time proportional to the nonzeros of \code{X} (a few per +#' row). +#' +#' \strong{What this implementation actually does.} The product is +#' currently written as a dense \code{coefficient_matrix \%*\% +#' search_direction}, which multiplies every entry, including the zero +#' off-diagonal blocks and the redundant lower-left block. As written, +#' each iteration therefore costs O(n^2) and a solve costs O(k * n^2). +#' This is still cheaper than an O(n^3) factorization, and it is +#' negligible at per-protein sizes. The O(k * n) cost described above +#' requires replacing that product with one that uses the structure - +#' either a sparse/symmetric matrix class or the factored form +#' \code{-X' (w * (X p))}. +#' #' @param coefficient_matrix symmetric positive (semi-)definite matrix, #' e.g. the Hessian/information matrix from a Newton step. #' @param right_hand_side vector the system is solved against, e.g. the diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 37094b64..2049efd0 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -413,6 +413,99 @@ #' needs, so it is a drop-in replacement anywhere \code{.fitSurvival}'s #' result is used. #' +#' @section Maximum likelihood estimation: +#' \strong{The model.} Each row i has a log-intensity \code{y_i}, a row +#' \code{x_i} of \code{design_matrix}, and linear predictor +#' \code{mu_i = x_i' beta}. The Gaussian AFT model says +#' \preformatted{ +#' y_i = mu_i + sigma * eps_i, eps_i ~ N(0, 1) +#' } +#' so each true log-intensity is normally distributed around its +#' prediction with a common standard deviation \code{sigma} (the fitted +#' \code{scale}). The unknowns are \code{theta = (beta, log sigma)}; +#' \code{sigma} is estimated on the log scale so that Newton steps are +#' unconstrained and can never produce a negative standard deviation. +#' Write \code{z_i = (y_i - mu_i) / sigma} for the standardized distance +#' from the prediction, \code{phi} for the standard normal density +#' (\code{dnorm}), and \code{Phi} for its CDF (\code{pnorm}). +#' +#' \strong{The objective: density for observed rows, CDF for censored +#' rows.} Maximum likelihood picks the \code{theta} under which the data we +#' saw were most probable. What we "saw" differs by row type +#' (\code{exact_indicator}): +#' \itemize{ +#' \item An \emph{observed} (exact) row has a known value, so it +#' contributes the normal density evaluated at that value: +#' \code{L_i = (1 / sigma) phi(z_i)}. +#' \item A \emph{censored} row is one whose intensity fell below the +#' detection limit. Its true value is unknown; all we know is that it lies +#' somewhere below the threshold \code{c_i} (which +#' \code{.setCensoredByThreshold} has substituted in as \code{y_i}). The +#' honest contribution is therefore the total probability of landing +#' anywhere below that threshold - the normal CDF: +#' \code{L_i = P(Y_i <= c_i) = Phi((c_i - mu_i) / sigma)}. +#' } +#' Taking logs and summing over rows gives the objective that is maximized: +#' \preformatted{ +#' l(theta) = sum_{observed} [ log phi(z_i) - log sigma ] +#' + sum_{censored} log Phi(z_i) +#' } +#' (the first sum is, up to a constant, ordinary least squares; the second +#' is what pulls \code{mu_i} and \code{sigma} toward values that make the +#' censored rows plausibly low). If a censored row's \code{mu_i} is well +#' above its threshold, \code{Phi(z_i)} is tiny and \code{l} is heavily +#' penalized - so the fit, and the imputed values later predicted from it, +#' respect the information that those rows were below the limit, rather +#' than ignoring them or treating the threshold as an exact value. +#' +#' \strong{Setting the derivatives to zero.} The maximum is where the score +#' (gradient of \code{l}) is zero. By the chain rule through +#' \code{mu_i = x_i' beta}, each row only needs its derivatives with respect +#' to \code{mu_i} and \code{log sigma} (computed by +#' \code{.aftGaussianDerivatives}): +#' \preformatted{ +#' observed: d l_i / d mu_i = z_i / sigma +#' censored: d l_i / d mu_i = -phi(z_i) / (sigma Phi(z_i)) +#' } +#' The observed term is the usual least-squares residual pull; the censored +#' term (an inverse Mills ratio) always pushes \code{mu_i} down, strongly +#' when the prediction sits above the threshold and negligibly when it is +#' already well below. These assemble into the score +#' (\code{.buildAFTGradient}), with \code{d} the vector of +#' \code{d l_i / d mu_i}: +#' \preformatted{ +#' U(theta) = [ X' d ] (gradient wrt beta) +#' [ sum_i d l_i / d log sigma ] (gradient wrt log sigma) +#' } +#' There is no closed-form root because of the \code{Phi} terms, so the +#' root is found iteratively. +#' +#' \strong{Newton-Raphson.} Expanding the score to first order around the +#' current guess, \code{U(theta + step) ~ U(theta) - I(theta) step}, and +#' setting it to zero gives the Newton step +#' \preformatted{ +#' I(theta) step = U(theta), theta_new = theta + step +#' } +#' where \code{I = -d^2 l / d theta d theta'} is the observed information +#' matrix (\code{.buildAFTInformationMatrix}), with blocks +#' \preformatted{ +#' I = - [ X' W X X' v ] W = diag(d^2 l_i / d mu_i^2) +#' [ v' X sum_i s_i ] v_i = d^2 l_i / d mu_i d log sigma +#' s_i = d^2 l_i / d (log sigma)^2 +#' } +#' This linear system is the part solved by \code{.cgSolve} (see its +#' documentation for the conjugate-gradient math). Starting values come +#' from ordinary least squares that ignores censoring. If a step fails to +#' increase \code{l} (or produces non-finite values), the candidate is +#' pulled toward the current guess, \code{(candidate + 2 * current) / 3}, +#' cutting the step to a third each time until \code{l} improves; if \code{I} is not positive definite (so the +#' Newton direction might not point uphill), the Gauss-Newton matrix +#' \code{sum_i g_i g_i'} of per-row score contributions is used instead, +#' which is positive semi-definite by construction. Iteration stops when \code{l} changes by less than +#' \code{convergence_tolerance}, and the inverse of the final information +#' matrix is returned as the coefficient variance-covariance matrix, as +#' \code{survreg} does. +#' #' @param input data.table, the same shape \code{.fitSurvival} expects. #' @param aft_iterations maximum number of log-likelihood evaluations the #' fit may spend. Newton-Raphson iterations and the step-halvings used to From 560a75577c98b8a8858c3fbb489e2aa90da02e93 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sat, 26 Sep 2026 19:23:08 -0400 Subject: [PATCH 25/30] update docs around n being number of entries in matrix --- R/utils_cgsolve.R | 76 +++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index 8b9ade64..db217896 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -168,33 +168,40 @@ #' \code{positive_definite = FALSE}. #' #' @section Computational cost: -#' \strong{Cost per solve is k matrix-vector products.} Each iteration -#' does exactly one product \code{A p} plus a fixed number of length-n dot -#' products and vector updates (O(n)). Over k iterations the total is +#' In this section, n is the number of entries in +#' \code{coefficient_matrix} - for m unknowns, \code{n = m^2} - not the +#' number of unknowns used in the sections above. +#' +#' \strong{A solve costs O(k * n): linear in the size of the matrix.} Each +#' iteration does exactly one product \code{A p}. Computing +#' \code{coefficient_matrix \%*\% search_direction} visits every entry of +#' \code{A} once (one multiply and one add each), so it costs O(n). The +#' rest of the iteration is a fixed number of length-m dot products and +#' vector updates, O(m) = O(sqrt(n)), which the product dominates. Over k +#' iterations the total is #' \preformatted{ -#' O(k * (cost of one A p + n)) +#' O(k * (n + m)) = O(k * n) #' } -#' Nothing here factorizes \code{A} (a Cholesky factorization costs -#' O(n^3)) or builds any matrix; \code{A} is only ever multiplied by a -#' vector. When a product \code{A p} costs O(n) - proportional to the -#' number of nonzero entries of \code{A} - a full solve costs O(k * n): -#' linear in the problem size for a fixed number of iterations. +#' \code{A} is never factorized or modified; it is only ever read, once per +#' iteration. By comparison, a Cholesky factorization costs O(m^3) = +#' O(n^1.5) regardless of how quickly the problem could converge, so CG +#' wins whenever k is small relative to m. #' #' \strong{k stays small on MSstats data.} Section 6 above shows k is -#' governed by the number of eigenvalue clusters, not by n. The AFT -#' information matrix is dominated by its diagonal (each parameter's own -#' curvature is much larger than its cross-terms), so Jacobi -#' preconditioning pulls most of the spectrum of \code{M^-1 A} to 1, with -#' only a few outlying eigenvalues. For example, in a simulated -#' 12-feature x 10-run protein (n = 22 unknowns, 20\% censored), 14 of the +#' governed by the number of eigenvalue clusters, not by the number of +#' unknowns. The AFT information matrix is dominated by its diagonal (each +#' parameter's own curvature is much larger than its cross-terms), so +#' Jacobi preconditioning pulls most of the spectrum of \code{M^-1 A} to +#' 1, with only a few outlying eigenvalues. For example, in a simulated +#' 12-feature x 10-run protein (m = 22 unknowns, 20\% censored), 14 of the #' 22 preconditioned eigenvalues lie in [0.9, 1.1], the condition number #' drops from about 234 to about 103, and CG reaches tolerance in 12 #' iterations instead of 16. Because the bulk cluster is handled in a few #' steps, and the remaining iterations are spent on the few outliers, k -#' grows much more slowly than n. +#' grows much more slowly than m. #' -#' \strong{Symmetry and sparsity: only the upper-right block carries real -#' work.} For the \code{~ FEATURE + RUN} model, every row of the design +#' \strong{Symmetry and sparsity: the real work is in the upper-right +#' block.} For the \code{~ FEATURE + RUN} model, every row of the design #' matrix \code{X} has exactly one feature indicator and one run indicator. #' In \code{A = -X' W X} (plus the log-scale row and column), this means: #' \itemize{ @@ -213,25 +220,22 @@ #' A = [ D_F C ] A p = [ D_F p_F + C p_R ] #' [ C' D_R ] [ C' p_F + D_R p_R ] #' } -#' with \code{D_F} and \code{D_R} diagonal: two elementwise scalings plus -#' one pass over \code{C}, used once as-is and once transposed. The -#' nonzero count is about \code{F + R + 2 * (number of feature/run cells)}, -#' i.e. proportional to the number of observations, rather than the -#' \code{n^2} entries of a dense matrix. Equivalently, -#' \code{A p = -X' (w * (X p))} can be computed without forming \code{A} -#' at all, in time proportional to the nonzeros of \code{X} (a few per -#' row). +#' with \code{D_F} and \code{D_R} diagonal. The only real computation in +#' each product is two elementwise scalings plus one pass over \code{C}, +#' used once as-is and once transposed. Everything else in the n entries is +#' either zero or a mirror image of \code{C}, so the number of distinct +#' nonzero values is about \code{F + R + (number of feature/run cells)}, +#' well below n. The dense product still visits all n entries, which is +#' what the O(k * n) bound counts. A product written against this +#' structure could skip the zeros and reuse \code{C} for both halves, but +#' at per-protein sizes the dense product is already cheap. #' -#' \strong{What this implementation actually does.} The product is -#' currently written as a dense \code{coefficient_matrix \%*\% -#' search_direction}, which multiplies every entry, including the zero -#' off-diagonal blocks and the redundant lower-left block. As written, -#' each iteration therefore costs O(n^2) and a solve costs O(k * n^2). -#' This is still cheaper than an O(n^3) factorization, and it is -#' negligible at per-protein sizes. The O(k * n) cost described above -#' requires replacing that product with one that uses the structure - -#' either a sparse/symmetric matrix class or the factored form -#' \code{-X' (w * (X p))}. +#' The same structure is why the Jacobi preconditioner works well here. +#' The diagonal blocks are exactly diagonal, so scaling by \code{diag(A)} +#' turns them into identity blocks, and the preconditioned matrix is the +#' identity plus only the scaled \code{C} coupling (and the border). Its +#' eigenvalues therefore sit near 1, spread only as far as that coupling +#' pushes them - the clustering that keeps k small. #' #' @param coefficient_matrix symmetric positive (semi-)definite matrix, #' e.g. the Hessian/information matrix from a Newton step. From e62c4ee6edab1ed40d4808c8008e5a460000962b Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sun, 27 Sep 2026 12:39:43 -0400 Subject: [PATCH 26/30] update documentation for CG-solve --- R/utils_cgsolve.R | 286 ++++++++++++---------------------------------- 1 file changed, 70 insertions(+), 216 deletions(-) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index db217896..d97dcf8d 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -1,52 +1,21 @@ -#' Solve a symmetric positive (semi-)definite linear system via conjugate -#' gradient -#' -#' A minimal, single right-hand-side conjugate gradient solver, used as the -#' Newton-step linear solve in \code{.fitSurvivalCG}. Modeled on -#' \code{lfe::cgsolve}, stripped down to a single dense matrix and a single -#' right-hand-side vector (no multi-column batching, no \code{Matrix}-package -#' or operator/closure dispatch - neither is needed for the small, dense AFT -#' information matrices this is used on). Optionally applies a Jacobi -#' (inverse-diagonal) preconditioner, which \code{lfe::cgsolve} does not -#' support at all. -#' -#' Iteration always starts from the zero vector and stops once the true -#' (unpreconditioned) residual is below -#' \code{1e-8 * max(norm(right_hand_side), 1)} - relative for large -#' right-hand sides, absolute (\code{1e-8}) for small ones - so the -#' tolerance means the same thing whether or not -#' \code{use_jacobi_preconditioner} is set. The absolute floor is -#' intentional: the caller (\code{.fitSurvivalCG}) passes a gradient, so a -#' right-hand side this small means the Newton iteration has already -#' converged, and \code{.fitSurvivalCG} judges convergence by the change in -#' log-likelihood rather than by the step returned here. In exact arithmetic, -#' conjugate gradient converges within \code{nrow(coefficient_matrix)} -#' steps, but rounding error erodes that guarantee as the system grows, so -#' up to \code{10 * nrow(coefficient_matrix)} steps are allowed. -#' -#' @section The math behind conjugate gradient: -#' Notation (with the variable each symbol lives in): \code{A} is -#' \code{coefficient_matrix} (n x n, symmetric positive definite), \code{b} -#' is \code{right_hand_side}, \code{x_k} is \code{solution} after k steps, -#' \code{r_k = b - A x_k} is \code{residual}, \code{M^-1} is the -#' preconditioner (\code{diag(A)^-1} for Jacobi, the identity otherwise), -#' \code{z_k = M^-1 r_k} is \code{preconditioned_residual}, and \code{p_k} -#' is \code{search_direction}. A prime (') denotes transpose, so -#' \code{u'v} is a dot product, computed here as \code{sum(u * v)}. -#' -#' \strong{1. Solving Ax = b as a minimization.} Because \code{A} is -#' symmetric positive definite, the quadratic -#' \preformatted{ -#' phi(x) = (1/2) x'A x - b'x -#' } -#' is a bowl with a unique minimum. Its gradient is -#' \code{grad phi(x) = A x - b = -r}, so setting the derivative to zero -#' gives exactly \code{A x = b}: the minimizer of \code{phi} is the -#' solution, and the residual is the negative gradient (the steepest -#' downhill direction). -#' -#' \strong{2. One iteration, written as matrix operations.} Starting from -#' \code{x_0 = 0}, \code{r_0 = b}, \code{z_0 = M^-1 r_0}, +#' Solve a system of linear equations Ax = b with the conjugate gradient method. +#' +#' @section Conjugate gradient is similar to gradient descent, except with +#' how it picks its search direction. +#' +#' Conjugate gradient is an iterative method that solves for "x" in Ax=b. +#' The method reformulates the linear system as an optimization problem where +#' one attempts to minimize Ax^2 - bx. Similar to gradient descent, the solution +#' is initialized at some arbitrary point, then the method iteratively descends +#' toward the optimal solution. But as opposed to gradient descent, which +#' moves in the direction of steepest descent, conjugate gradient picks a +#' different search direction. +#' +#' @section Conjugate gradient search direction is determined based on a linear +#' transformation of the previous iteration's search direction. +#' +#' \strong{Pseudocode for one iteration, written as matrix operations.} +#' Starting from \code{x_0 = 0}, \code{r_0 = b}, \code{z_0 = M^-1 r_0}, #' \code{p_0 = z_0}, each step computes #' \preformatted{ #' q_k = A p_k (the only matrix-vector @@ -58,184 +27,69 @@ #' without a new product) #' z_k+1 = M^-1 r_k+1 (elementwise scaling for #' Jacobi) -#' beta_k = (r_k+1' z_k+1) / (r_k' z_k) +#' beta_k = (r_k+1' z_k+1) / (r_k' z_k) (guarantees p_k A p_k+1 = 0, +#' i.e. A-conjugacy) #' p_k+1 = z_k+1 + beta_k p_k #' } -#' In the code, \code{q_k} is \code{matrix_times_search_direction}, -#' \code{p_k' q_k} is \code{curvature}, \code{alpha_k} is -#' \code{step_length}, and \code{r_k' z_k} is -#' \code{residual_dot_preconditioned_residual}. The per-step cost is one -#' n x n matrix-vector product (O(n^2)) plus a handful of O(n) dot products -#' and vector updates, and only the current \code{x}, \code{r}, \code{z}, -#' and \code{p} are kept - no earlier directions need to be stored. -#' -#' \strong{3. Where alpha comes from: residuals orthogonal to the step -#' taken.} Given a direction \code{p_k}, pick the step that minimizes -#' \code{phi} along that line by setting the derivative to zero: +#' +#' @section If A is a (p x p) matrix, conjugate gradient is guaranteed to +#' converge in p iterations +#' +#' \strong{Residuals are orthogonal to the step taken.} Given a direction +#' \code{p_k}, CG picks the step size that minimizes the objective along the +#' search direction by setting the derivative to zero with respect to alpha: #' \preformatted{ #' d/d alpha phi(x_k + alpha p_k) #' = p_k' (A x_k + alpha A p_k - b) #' = -p_k' r_k + alpha p_k' A p_k = 0 #' => alpha_k = (p_k' r_k) / (p_k' A p_k) +#' +#' x_k+1 = x_k + alpha_k p_k +#' r_k+1 = b - A x_k+1 = r_k - alpha_k A p_k +#' p_k' r_k+1 = p_k' r_k - alpha_k p_k' A p_k +#' = p_k' r_k - (p_k' r_k / p_k' A p_k) p_k' A p_k +#' = p_k' r_k - p_k' r_k +#' = 0 #' } -#' The zero-derivative condition is the same statement as -#' \code{p_k' r_k+1 = 0}: the new residual (the new downhill direction) is -#' orthogonal to the direction just searched, so there is nothing left to -#' gain along \code{p_k}. Since \code{p_k = z_k + beta_k-1 p_k-1} and -#' \code{p_k-1' r_k = 0} by the same argument one step earlier, -#' \code{p_k' r_k = z_k' r_k}, which is the numerator used in the code. #' -#' \strong{4. Where beta comes from: A-conjugate directions.} Steepest -#' descent (always stepping along \code{r_k}) also does an exact line -#' search, but a later step can undo progress made along an earlier -#' direction, so it zig-zags across narrow valleys. Conjugate gradient -#' avoids this by choosing each new direction to be \emph{A-conjugate} to -#' the previous one: +#' \strong{Residuals are orthogonal to all other residuals.} +#' +#' Dotting \code{r_k+1 = r_k - alpha_k A p_k} with \code{p_k-1} instead: +#' #' \preformatted{ -#' p_k+1' A p_k = 0 -#' => (z_k+1 + beta_k p_k)' A p_k = 0 -#' => beta_k = -(z_k+1' A p_k) / (p_k' A p_k) +#' p_k-1' r_k = 0 +#' p_k-1' A p_k = 0 (by design of beta to ensure A-conjugacy) +#' p_k-1' r_k+1 = p_k-1' r_k - alpha_k p_k-1' A p_k = 0 +#' +#' r_k = p_k - beta_k-1 p_k-1 +#' r_k' r_k+1 = (p_k - beta_k-1 p_k-1)' r_k+1 +#' = p_k' r_k+1 - beta_k-1 (p_k-1' r_k+1) +#' = 0 - beta_k-1 (0) +#' = 0 #' } -#' Substituting \code{A p_k = (r_k - r_k+1) / alpha_k} (from the residual -#' update) together with the orthogonality \code{z_k' r_k+1 = 0} collapses -#' this to \code{beta_k = (r_k+1' z_k+1) / (r_k' z_k)}, a ratio of two -#' dot products that are already on hand. \code{p_i' A p_j = 0} says the -#' directions are orthogonal under the inner product -#' \code{_A = u' A v} - i.e. they would be ordinary perpendicular -#' vectors if space were stretched by \code{A} so the elliptical contours -#' of \code{phi} became circles. By induction the short recurrence gives -#' this for all pairs, not just neighbours: -#' \preformatted{ -#' p_i' A p_j = 0 for all i != j (directions A-conjugate) -#' r_i' M^-1 r_j = 0 for all i != j (residuals orthogonal; plain -#' r_i' r_j = 0 when M = I) -#' } -#' -#' \strong{5. Verifying optimality with derivatives, and why this is -#' fast.} After k steps \code{x_k = alpha_0 p_0 + ... + alpha_k-1 p_k-1}. -#' Take the derivative of \code{phi} with respect to the coefficient on -#' any earlier direction \code{p_j} (j < k): -#' \preformatted{ -#' d phi / d c_j = p_j' (A x_k - b) = -p_j' r_k = 0 -#' } -#' Every one of these partial derivatives is zero, so \code{x_k} is not -#' just the best point on the latest line but the exact minimizer of -#' \code{phi} over the whole subspace \code{span(p_0, ..., p_k-1)} - which -#' equals the Krylov subspace \code{span(r_0, A r_0, ..., A^(k-1) r_0)} -#' (with \code{M^-1} folded in when preconditioned). Conjugacy is what -#' makes this possible: writing the exact answer as -#' \code{x* = sum_j alpha_j p_j} and multiplying \code{A x* = b} on the -#' left by \code{p_j'} kills every cross term, leaving -#' \code{alpha_j = (p_j' b) / (p_j' A p_j)} - each coefficient is -#' determined independently, so each direction is solved once and never -#' revisited. With n independent directions available, exact arithmetic -#' reaches the solution in at most n steps. -#' -#' \strong{6. Why clustered eigenvalues make it faster still.} Because -#' \code{x_k} is optimal over the Krylov subspace, its error -#' \code{e_k = x* - x_k} can be written \code{e_k = P_k(A) e_0} for the -#' degree-k polynomial \code{P_k} with \code{P_k(0) = 1} that minimizes the -#' A-norm of the error. Expanding \code{e_0 = sum_i c_i v_i} in the -#' eigenvectors \code{v_i} of \code{A} (eigenvalues \code{lambda_i}): -#' \preformatted{ -#' ||e_k||_A^2 = sum_i lambda_i c_i^2 P_k(lambda_i)^2 -#' ||e_k||_A / ||e_0||_A <= min over P_k of max_i |P_k(lambda_i)| -#' } -#' so convergence depends on how many \emph{distinct groups} of -#' eigenvalues there are, not on n. If the eigenvalues fall into m tight -#' clusters, a degree-m polynomial with one root near the centre of each -#' cluster is small at every eigenvalue, and CG is essentially done after -#' about m steps. Intuitively, within a cluster \code{A} acts almost like a -#' single scalar, so the curvature \code{p' A p / p' p} - and hence the -#' step length \code{alpha} - is nearly the same for every eigen-direction -#' in that cluster; one step of that size removes the error along all of -#' those directions at once instead of one at a time (in the extreme case -#' \code{A = c I}, a single step solves the system exactly). When the -#' spectrum is spread out instead, the standard bound -#' \preformatted{ -#' ||e_k||_A <= 2 ((sqrt(kappa) - 1) / (sqrt(kappa) + 1))^k ||e_0||_A, -#' kappa = lambda_max / lambda_min -#' } -#' applies. This is the motivation for the Jacobi preconditioner: CG is -#' effectively run on \code{M^-1 A}, and for a diagonally dominant AFT -#' information matrix dividing by the diagonal pulls the eigenvalues -#' toward 1 - clustering them and shrinking \code{kappa}. -#' -#' Finally, the curvature \code{p_k' A p_k} must be positive for -#' \code{phi} to have a minimum along \code{p_k}; if it is zero, negative, -#' or non-finite, \code{A} is not positive definite along that direction, -#' the line search has no minimizer, and the loop stops and reports -#' \code{positive_definite = FALSE}. -#' -#' @section Computational cost: -#' In this section, n is the number of entries in -#' \code{coefficient_matrix} - for m unknowns, \code{n = m^2} - not the -#' number of unknowns used in the sections above. -#' -#' \strong{A solve costs O(k * n): linear in the size of the matrix.} Each -#' iteration does exactly one product \code{A p}. Computing -#' \code{coefficient_matrix \%*\% search_direction} visits every entry of -#' \code{A} once (one multiply and one add each), so it costs O(n). The -#' rest of the iteration is a fixed number of length-m dot products and -#' vector updates, O(m) = O(sqrt(n)), which the product dominates. Over k -#' iterations the total is -#' \preformatted{ -#' O(k * (n + m)) = O(k * n) -#' } -#' \code{A} is never factorized or modified; it is only ever read, once per -#' iteration. By comparison, a Cholesky factorization costs O(m^3) = -#' O(n^1.5) regardless of how quickly the problem could converge, so CG -#' wins whenever k is small relative to m. -#' -#' \strong{k stays small on MSstats data.} Section 6 above shows k is -#' governed by the number of eigenvalue clusters, not by the number of -#' unknowns. The AFT information matrix is dominated by its diagonal (each -#' parameter's own curvature is much larger than its cross-terms), so -#' Jacobi preconditioning pulls most of the spectrum of \code{M^-1 A} to -#' 1, with only a few outlying eigenvalues. For example, in a simulated -#' 12-feature x 10-run protein (m = 22 unknowns, 20\% censored), 14 of the -#' 22 preconditioned eigenvalues lie in [0.9, 1.1], the condition number -#' drops from about 234 to about 103, and CG reaches tolerance in 12 -#' iterations instead of 16. Because the bulk cluster is handled in a few -#' steps, and the remaining iterations are spent on the few outliers, k -#' grows much more slowly than m. -#' -#' \strong{Symmetry and sparsity: the real work is in the upper-right -#' block.} For the \code{~ FEATURE + RUN} model, every row of the design -#' matrix \code{X} has exactly one feature indicator and one run indicator. -#' In \code{A = -X' W X} (plus the log-scale row and column), this means: -#' \itemize{ -#' \item the feature-feature block is \emph{diagonal} - two different -#' features never appear in the same row, so their cross-term is zero; -#' \item the run-run block is likewise \emph{diagonal}; -#' \item the only dense off-diagonal content is the feature x run block -#' \code{C} in the upper-right corner (one entry per feature/run cell -#' that has data), together with the intercept and log-scale border -#' rows; -#' \item by symmetry the lower-left block is \code{C'}, so it carries no -#' new information. -#' } -#' Ignoring the border, the product therefore reduces to -#' \preformatted{ -#' A = [ D_F C ] A p = [ D_F p_F + C p_R ] -#' [ C' D_R ] [ C' p_F + D_R p_R ] -#' } -#' with \code{D_F} and \code{D_R} diagonal. The only real computation in -#' each product is two elementwise scalings plus one pass over \code{C}, -#' used once as-is and once transposed. Everything else in the n entries is -#' either zero or a mirror image of \code{C}, so the number of distinct -#' nonzero values is about \code{F + R + (number of feature/run cells)}, -#' well below n. The dense product still visits all n entries, which is -#' what the O(k * n) bound counts. A product written against this -#' structure could skip the zeros and reuse \code{C} for both halves, but -#' at per-protein sizes the dense product is already cheap. -#' -#' The same structure is why the Jacobi preconditioner works well here. -#' The diagonal blocks are exactly diagonal, so scaling by \code{diag(A)} -#' turns them into identity blocks, and the preconditioned matrix is the -#' identity plus only the scaled \code{C} coupling (and the border). Its -#' eigenvalues therefore sit near 1, spread only as far as that coupling -#' pushes them - the clustering that keeps k small. +#' +#' So each new residual is orthogonal to the previous one as well. Since in +#' a p-dimensional system, there can only be p orthogonal residuals, CG +#' is guaranteed to converge in p-iterations. +#' +#' @section Time complexity is empirically linear with respect to the number +#' of entries in the coefficient matrix +#' +#' The time complexity of CG is O(kn), where n is the number of entries in +#' matrix A, and k is the number of iterations. As shown earlier, k is +#' guaranteed to be at most the number of rows in matrix A, i.e. n^0.5. +#' +#' But we can ensure k is small and near a constant given certain data +#' constraints. In the case of MSstats, the following intuition leads +#' k to be empirically small: +#' +#' 1. MSstats Hessian matrix contains zeros on +#' (run x run) or (feature x feature) entries +#' 2. Sparsity makes it more likely diagonals of a matrix dominate +#' 3. Because diagonals more likely dominate, scaling the matrix by the inverse +#' of the diagonals (Jacobi preconditioning) makes the eigenvalues of the +#' resultant matrix to cluster well +#' 4. Well-clustered eigenvalues reduces the number of iterations needed #' #' @param coefficient_matrix symmetric positive (semi-)definite matrix, #' e.g. the Hessian/information matrix from a Newton step. From 8c1119aa82d28ef48c7c0191550ae32ab3fafa06 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sun, 27 Sep 2026 12:57:13 -0400 Subject: [PATCH 27/30] remove some unnecessary documentation --- R/utils_imputation.R | 42 ++++++++---------------------------------- 1 file changed, 8 insertions(+), 34 deletions(-) diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 2049efd0..faa8dddb 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -3,13 +3,9 @@ #' MSstats fits an accelerated-failure-time (AFT) model per protein to #' impute left-censored values, and predictors are chosen based on how much #' information is actually available: whether this is a labeled (SRM) -#' experiment with a reference channel (\code{ref_covariate}), whether -#' there is more than one feature to estimate a \code{FEATURE} effect for, -#' and whether there are enough uncensored observations to estimate that -#' effect at all. Both \code{.fitSurvival} (Cholesky-based, via -#' \code{survival::survreg}) and \code{.fitSurvivalCG} (conjugate-gradient -#' based) share this selection logic, so the two solvers always fit the -#' same model and differ only in how the Newton step is solved. +#' experiment (\code{ref_covariate}), whether there is more than one feature +#' to estimate a \code{FEATURE} effect for, and whether there are enough +#' uncensored observations to estimate that effect at all. #' #' @param input data.table with columns \code{newABUNDANCE}, \code{cen}, #' \code{RUN}, \code{FEATURE}, \code{LABEL}, and (for labeled experiments) @@ -53,6 +49,8 @@ } } +#' Fit an AFT survival model with SurvReg dependency +#' #' @param input data.table with the columns \code{.buildAFTFormula} needs. #' @param aft_iterations maximum number of iterations for AFT model fitting. #' @param verbose if \code{TRUE}, \code{message()} the problem size @@ -92,23 +90,6 @@ #' Per-observation log-likelihood and derivatives for a Gaussian AFT model #' -#' Computes what a Newton-Raphson step needs at the current parameter -#' guess: the log-likelihood, its first derivative with respect to the -#' linear predictor and to the log of the scale parameter, and the -#' corresponding second derivatives - all summed/assembled later into the -#' score vector and information matrix by \code{.fitSurvivalCG}. This only -#' covers the two cases MSstats' AFT imputation actually uses: an exact -#' (uncensored) observation, or one left-censored below a detection-limit -#' ceiling (\code{Surv(y, cen, type = "left")} with \code{cen == 0}). -#' -#' The formulas are transcribed term-for-term from \code{survival}'s own -#' C implementation (\code{survregc1.c}'s \code{gauss_d} function and its -#' "exact"/"left censored" cases) rather than re-derived by hand, since a -#' hand re-derivation is an easy place to introduce a sign error; this -#' function's correctness is instead checked against numerical -#' differentiation of the log-likelihood (see -#' \code{test_utils_imputation_cg.R}). -#' #' @param linear_predictor current linear predictor #' (\code{model_matrix \%*\% coefficients}). #' @param log_scale current log of the scale parameter. @@ -244,9 +225,6 @@ #' Evaluate the Gaussian AFT log-likelihood and derivatives at a #' parameter guess #' -#' Thin wrapper around \code{.aftGaussianDerivatives} that forms the -#' linear predictor from \code{design_matrix} and \code{coefficients}. -#' #' @param design_matrix model matrix of the AFT fit. #' @param coefficients current regression coefficients. #' @param log_scale current log of the scale parameter. @@ -266,7 +244,7 @@ observed_value, exact_indicator) } -#' Assemble the AFT score vector +#' Assemble the AFT gradient vector #' #' @param design_matrix model matrix of the AFT fit. #' @param derivatives output of \code{.aftGaussianDerivatives}. @@ -282,7 +260,7 @@ sum(derivatives$gradient_wrt_log_scale)) } -#' Assemble the AFT observed information matrix +#' Assemble the AFT Hessian matrix #' #' @param design_matrix model matrix of the AFT fit. #' @param derivatives output of \code{.aftGaussianDerivatives}. @@ -323,7 +301,7 @@ #' Gauss-Newton (outer-product-of-gradients) approximation to the AFT #' information matrix #' -#' Always positive semi-definite, so it is used as a fallback when the +#' A fallback when the #' observed information matrix is not positive definite. #' #' @param design_matrix model matrix of the AFT fit. @@ -360,10 +338,6 @@ #' Solve for one AFT Newton-Raphson step with conjugate gradient #' -#' Solves \code{information_matrix \%*\% step = gradient}; if the -#' information matrix turns out not to be positive definite, re-solves -#' against the Gauss-Newton approximation instead. -#' #' @param design_matrix model matrix of the AFT fit. #' @param information_matrix output of \code{.buildAFTInformationMatrix}. #' @param derivatives output of \code{.aftGaussianDerivatives}. From 77d7e153a4513e28d387ecf8d1055df73e4ec719 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sun, 27 Sep 2026 13:01:12 -0400 Subject: [PATCH 28/30] fix docs w.r.t. negative hessian --- R/utils_cgsolve.R | 4 ++-- R/utils_imputation.R | 56 +++++++++++++++++++++++--------------------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/R/utils_cgsolve.R b/R/utils_cgsolve.R index d97dcf8d..3d8eefd8 100644 --- a/R/utils_cgsolve.R +++ b/R/utils_cgsolve.R @@ -92,13 +92,13 @@ #' 4. Well-clustered eigenvalues reduces the number of iterations needed #' #' @param coefficient_matrix symmetric positive (semi-)definite matrix, -#' e.g. the Hessian/information matrix from a Newton step. +#' e.g. the negative Hessian from a Newton step. #' @param right_hand_side vector the system is solved against, e.g. the #' gradient/score vector from a Newton step. #' @param use_jacobi_preconditioner if \code{TRUE}, precondition with the #' inverse of \code{coefficient_matrix}'s own diagonal - cheap to apply, #' and often enough to cut down the number of iterations needed when the -#' diagonal dominates (as it typically does for an AFT information matrix, +#' diagonal dominates (as it typically does for an AFT negative Hessian, #' where each parameter's own curvature tends to be much larger than its #' cross-terms with the other parameters). Defaults to \code{FALSE}, which #' reduces exactly to plain (unpreconditioned) conjugate gradient. diff --git a/R/utils_imputation.R b/R/utils_imputation.R index faa8dddb..921d0387 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -260,7 +260,7 @@ sum(derivatives$gradient_wrt_log_scale)) } -#' Assemble the AFT Hessian matrix +#' Assemble the negative Hessian of the AFT log-likelihood #' #' @param design_matrix model matrix of the AFT fit. #' @param derivatives output of \code{.aftGaussianDerivatives}. @@ -270,7 +270,7 @@ #' #' @keywords internal #' @noRd -.buildAFTInformationMatrix = function(design_matrix, derivatives) { +.buildAFTNegativeHessian = function(design_matrix, derivatives) { regression_block = -crossprod( design_matrix, design_matrix * derivatives$second_derivative_wrt_linear_predictor) @@ -299,10 +299,10 @@ } #' Gauss-Newton (outer-product-of-gradients) approximation to the AFT -#' information matrix +#' negative Hessian #' #' A fallback when the -#' observed information matrix is not positive definite. +#' negative Hessian is not positive definite. #' #' @param design_matrix model matrix of the AFT fit. #' @param derivatives output of \code{.aftGaussianDerivatives}. @@ -339,7 +339,7 @@ #' Solve for one AFT Newton-Raphson step with conjugate gradient #' #' @param design_matrix model matrix of the AFT fit. -#' @param information_matrix output of \code{.buildAFTInformationMatrix}. +#' @param negative_hessian output of \code{.buildAFTNegativeHessian}. #' @param derivatives output of \code{.aftGaussianDerivatives}. #' @param gradient output of \code{.buildAFTGradient}. #' @param use_jacobi_preconditioner passed to \code{.cgSolve}. @@ -349,11 +349,11 @@ #' #' @keywords internal #' @noRd -.solveAFTNewtonStep = function(design_matrix, information_matrix, +.solveAFTNewtonStep = function(design_matrix, negative_hessian, derivatives, gradient, use_jacobi_preconditioner) { primary_solve = .cgSolveMufflingPDWarning( - information_matrix, gradient, + negative_hessian, gradient, use_jacobi_preconditioner = use_jacobi_preconditioner) if (primary_solve$positive_definite) { list(step = primary_solve$solution, @@ -379,7 +379,7 @@ #' only, chosen by the same \code{.buildAFTFormula} both solvers share), #' used when \code{aft_solver = "cg"}. It runs the same kind of #' Newton-Raphson iteration \code{survival::survreg} does - repeatedly -#' solving \code{information_matrix \%*\% step = gradient} for the next +#' solving \code{negative_hessian \%*\% step = gradient} for the next #' set of coefficients - but performs that linear solve with the #' conjugate-gradient routine \code{.cgSolve} instead of the Cholesky #' factorization \code{survreg} uses internally. The returned object is @@ -455,29 +455,31 @@ #' root is found iteratively. #' #' \strong{Newton-Raphson.} Expanding the score to first order around the -#' current guess, \code{U(theta + step) ~ U(theta) - I(theta) step}, and +#' current guess, \code{U(theta + step) ~ U(theta) + H(theta) step}, and #' setting it to zero gives the Newton step #' \preformatted{ -#' I(theta) step = U(theta), theta_new = theta + step +#' -H(theta) step = U(theta), theta_new = theta + step #' } -#' where \code{I = -d^2 l / d theta d theta'} is the observed information -#' matrix (\code{.buildAFTInformationMatrix}), with blocks +#' where \code{H = d^2 l / d theta d theta'} is the Hessian of the +#' log-likelihood, with blocks #' \preformatted{ -#' I = - [ X' W X X' v ] W = diag(d^2 l_i / d mu_i^2) -#' [ v' X sum_i s_i ] v_i = d^2 l_i / d mu_i d log sigma -#' s_i = d^2 l_i / d (log sigma)^2 +#' H = [ X' W X X' v ] W = diag(d^2 l_i / d mu_i^2) +#' [ v' X sum_i s_i ] v_i = d^2 l_i / d mu_i d log sigma +#' s_i = d^2 l_i / d (log sigma)^2 #' } +#' The negative Hessian \code{-H} (\code{.buildAFTNegativeHessian}) is +#' also known as the observed information matrix. #' This linear system is the part solved by \code{.cgSolve} (see its #' documentation for the conjugate-gradient math). Starting values come #' from ordinary least squares that ignores censoring. If a step fails to #' increase \code{l} (or produces non-finite values), the candidate is #' pulled toward the current guess, \code{(candidate + 2 * current) / 3}, -#' cutting the step to a third each time until \code{l} improves; if \code{I} is not positive definite (so the +#' cutting the step to a third each time until \code{l} improves; if \code{-H} is not positive definite (so the #' Newton direction might not point uphill), the Gauss-Newton matrix #' \code{sum_i g_i g_i'} of per-row score contributions is used instead, #' which is positive semi-definite by construction. Iteration stops when \code{l} changes by less than -#' \code{convergence_tolerance}, and the inverse of the final information -#' matrix is returned as the coefficient variance-covariance matrix, as +#' \code{convergence_tolerance}, and the inverse of the final negative +#' Hessian is returned as the coefficient variance-covariance matrix, as #' \code{survreg} does. #' #' @param input data.table, the same shape \code{.fitSurvival} expects. @@ -490,8 +492,8 @@ #' between iterations falls below this (matches the default #' \code{rel.tolerance} in \code{survival::survreg.control}). #' @param use_jacobi_preconditioner if \code{TRUE}, precondition every -#' conjugate-gradient solve with the inverse of the current information -#' matrix's own diagonal (see \code{.cgSolve}'s +#' conjugate-gradient solve with the inverse of the current negative +#' Hessian's own diagonal (see \code{.cgSolve}'s #' \code{use_jacobi_preconditioner}). This is what \code{aft_solver = #' "pcg"} enables, versus plain conjugate gradient for \code{"cg"}. #' @param verbose if \code{TRUE}, \code{message()} a line per @@ -557,10 +559,10 @@ iteration_start_time = Sys.time() gradient = .buildAFTGradient(design_matrix, current_fit) - information_matrix = - .buildAFTInformationMatrix(design_matrix, current_fit) + negative_hessian = + .buildAFTNegativeHessian(design_matrix, current_fit) newton_step = .solveAFTNewtonStep( - design_matrix, information_matrix, current_fit, gradient, + design_matrix, negative_hessian, current_fit, gradient, use_jacobi_preconditioner) elapsed_seconds = @@ -644,11 +646,11 @@ sum(cg_diagnostics$elapsed_seconds), converged)) } - final_information_matrix = - .buildAFTInformationMatrix(design_matrix, current_fit) + final_negative_hessian = + .buildAFTNegativeHessian(design_matrix, current_fit) variance_covariance_matrix = tryCatch( - solve(final_information_matrix), - error = function(e) MASS::ginv(final_information_matrix)) + solve(final_negative_hessian), + error = function(e) MASS::ginv(final_negative_hessian)) fitted_coefficients = coefficients names(fitted_coefficients) = colnames(design_matrix) From bb30fa336e3d9adb8fa0f9efce309315f120d13c Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sun, 27 Sep 2026 13:33:44 -0400 Subject: [PATCH 29/30] update docs --- NAMESPACE | 194 ++++++++++++++++++++++++------------------- R/utils_imputation.R | 150 +++++++++++++-------------------- 2 files changed, 167 insertions(+), 177 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 80be4853..5f156811 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -55,102 +55,124 @@ import(data.table) import(ggplot2) import(limma) import(lme4) -importFrom(BiocParallel,bpisup) -importFrom(BiocParallel,bplapply) -importFrom(BiocParallel,bpnworkers) -importFrom(BiocParallel,bpprogressbar) -importFrom(BiocParallel,bpstart) -importFrom(BiocParallel,bpstop) +importFrom(BiocParallel, + bpisup, + bplapply, + bpnworkers, + bpprogressbar, + bpstart, + bpstop +) importFrom(MASS,rlm) -importFrom(MSstatsConvert,DIANNtoMSstatsFormat) -importFrom(MSstatsConvert,DIAUmpiretoMSstatsFormat) -importFrom(MSstatsConvert,FragPipetoMSstatsFormat) -importFrom(MSstatsConvert,MSstatsBalancedDesign) -importFrom(MSstatsConvert,MSstatsClean) -importFrom(MSstatsConvert,MSstatsImport) -importFrom(MSstatsConvert,MSstatsLogsSettings) -importFrom(MSstatsConvert,MSstatsMakeAnnotation) -importFrom(MSstatsConvert,MSstatsPreprocess) -importFrom(MSstatsConvert,MZMinetoMSstatsFormat) -importFrom(MSstatsConvert,MaxQtoMSstatsFormat) -importFrom(MSstatsConvert,OpenMStoMSstatsFormat) -importFrom(MSstatsConvert,OpenSWATHtoMSstatsFormat) -importFrom(MSstatsConvert,PDtoMSstatsFormat) -importFrom(MSstatsConvert,ProgenesistoMSstatsFormat) -importFrom(MSstatsConvert,SkylinetoMSstatsFormat) -importFrom(MSstatsConvert,SpectronauttoMSstatsFormat) +importFrom(MSstatsConvert, + DIANNtoMSstatsFormat, + DIAUmpiretoMSstatsFormat, + FragPipetoMSstatsFormat, + MSstatsBalancedDesign, + MSstatsClean, + MSstatsImport, + MSstatsLogsSettings, + MSstatsMakeAnnotation, + MSstatsPreprocess, + MZMinetoMSstatsFormat, + MaxQtoMSstatsFormat, + OpenMStoMSstatsFormat, + OpenSWATHtoMSstatsFormat, + PDtoMSstatsFormat, + ProgenesistoMSstatsFormat, + SkylinetoMSstatsFormat, + SpectronauttoMSstatsFormat +) importFrom(Rcpp,sourceCpp) importFrom(RhpcBLASctl,blas_set_num_threads) -importFrom(data.table,as.data.table) -importFrom(data.table,data.table) -importFrom(data.table,fifelse) -importFrom(data.table,melt) -importFrom(data.table,rbindlist) -importFrom(data.table,setDT) -importFrom(data.table,setDTthreads) -importFrom(data.table,uniqueN) +importFrom(data.table, + as.data.table, + data.table, + fifelse, + melt, + rbindlist, + setDT, + setDTthreads, + uniqueN +) importFrom(ggrepel,geom_text_repel) importFrom(gplots,heatmap.2) -importFrom(grDevices,dev.off) -importFrom(grDevices,hcl) -importFrom(grDevices,pdf) -importFrom(graphics,axis) -importFrom(graphics,image) -importFrom(graphics,legend) -importFrom(graphics,mtext) -importFrom(graphics,par) -importFrom(graphics,plot) -importFrom(graphics,plot.new) -importFrom(graphics,title) -importFrom(htmltools,div) -importFrom(htmltools,save_html) -importFrom(htmltools,tagList) +importFrom(grDevices, + dev.off, + hcl, + pdf +) +importFrom(graphics, + axis, + image, + legend, + mtext, + par, + plot, + plot.new, + title +) +importFrom(htmltools, + div, + save_html, + tagList +) importFrom(limma,squeezeVar) importFrom(lme4,lmer) importFrom(marray,maPalette) importFrom(matter,SnowfastParam) importFrom(methods,is) -importFrom(parallel,clusterExport) -importFrom(parallel,makeCluster) -importFrom(parallel,parLapply) -importFrom(parallel,stopCluster) -importFrom(plotly,add_trace) -importFrom(plotly,ggplotly) -importFrom(plotly,layout) -importFrom(plotly,plot_ly) -importFrom(plotly,style) -importFrom(plotly,subplot) +importFrom(parallel, + clusterExport, + makeCluster, + parLapply, + stopCluster +) +importFrom(plotly, + add_trace, + ggplotly, + layout, + plot_ly, + style, + subplot +) importFrom(preprocessCore,normalize.quantiles) importFrom(rlang,.data) -importFrom(stats,dist) -importFrom(stats,dnorm) -importFrom(stats,fitted) -importFrom(stats,formula) -importFrom(stats,hclust) -importFrom(stats,lm) -importFrom(stats,lm.fit) -importFrom(stats,loess) -importFrom(stats,median) -importFrom(stats,model.frame) -importFrom(stats,model.matrix) -importFrom(stats,model.response) -importFrom(stats,na.omit) -importFrom(stats,p.adjust) -importFrom(stats,pnorm) -importFrom(stats,predict) -importFrom(stats,qbinom) -importFrom(stats,qnorm) -importFrom(stats,qt) -importFrom(stats,quantile) -importFrom(stats,resid) -importFrom(stats,residuals) -importFrom(stats,sd) -importFrom(stats,vcov) -importFrom(stats,xtabs) -importFrom(survival,Surv) -importFrom(survival,survreg) -importFrom(utils,combn) -importFrom(utils,sessionInfo) -importFrom(utils,setTxtProgressBar) -importFrom(utils,txtProgressBar) +importFrom(stats, + dist, + dnorm, + fitted, + formula, + hclust, + lm, + lm.fit, + loess, + median, + model.frame, + model.matrix, + model.response, + na.omit, + p.adjust, + pnorm, + predict, + qbinom, + qnorm, + qt, + quantile, + resid, + residuals, + sd, + vcov, + xtabs +) +importFrom(survival, + Surv, + survreg +) +importFrom(utils, + combn, + sessionInfo, + setTxtProgressBar, + txtProgressBar +) useDynLib(MSstats, .registration=TRUE) diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 921d0387..90745ddb 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -301,8 +301,7 @@ #' Gauss-Newton (outer-product-of-gradients) approximation to the AFT #' negative Hessian #' -#' A fallback when the -#' negative Hessian is not positive definite. +#' A fallback when the negative Hessian is not positive definite. #' #' @param design_matrix model matrix of the AFT fit. #' @param derivatives output of \code{.aftGaussianDerivatives}. @@ -372,40 +371,18 @@ } #' Fit a Gaussian, left-censored AFT model with a conjugate-gradient -#' Newton step -#' -#' An alternative to \code{.fitSurvival} for exactly the same imputation -#' model (Gaussian accelerated-failure-time regression, left-censoring -#' only, chosen by the same \code{.buildAFTFormula} both solvers share), -#' used when \code{aft_solver = "cg"}. It runs the same kind of -#' Newton-Raphson iteration \code{survival::survreg} does - repeatedly -#' solving \code{negative_hessian \%*\% step = gradient} for the next -#' set of coefficients - but performs that linear solve with the -#' conjugate-gradient routine \code{.cgSolve} instead of the Cholesky -#' factorization \code{survreg} uses internally. The returned object is -#' classed \code{"survreg"} and carries the fields \code{predict.survreg} -#' needs, so it is a drop-in replacement anywhere \code{.fitSurvival}'s -#' result is used. -#' -#' @section Maximum likelihood estimation: -#' \strong{The model.} Each row i has a log-intensity \code{y_i}, a row -#' \code{x_i} of \code{design_matrix}, and linear predictor -#' \code{mu_i = x_i' beta}. The Gaussian AFT model says -#' \preformatted{ -#' y_i = mu_i + sigma * eps_i, eps_i ~ N(0, 1) -#' } -#' so each true log-intensity is normally distributed around its -#' prediction with a common standard deviation \code{sigma} (the fitted -#' \code{scale}). The unknowns are \code{theta = (beta, log sigma)}; -#' \code{sigma} is estimated on the log scale so that Newton steps are -#' unconstrained and can never produce a negative standard deviation. -#' Write \code{z_i = (y_i - mu_i) / sigma} for the standardized distance -#' from the prediction, \code{phi} for the standard normal density -#' (\code{dnorm}), and \code{Phi} for its CDF (\code{pnorm}). +#' Newton step (rather than a cholesky solve) #' -#' \strong{The objective: density for observed rows, CDF for censored -#' rows.} Maximum likelihood picks the \code{theta} under which the data we -#' saw were most probable. What we "saw" differs by row type +#' @section Under the hood, the AFT model is fit with maximum likelihood +#' estimation, where the objective is a Gaussian density for observed rows and +#' CDF for censored rows. +#' +#' \code{phi} for the standard normal density +#' (\code{dnorm}), and \code{Phi} for its CDF (\code{pnorm}). +#' Maximum likelihood picks the set of parameter values \code{theta} +#' under which the data we saw were most probable. What we "saw" differs by +#' whether a row is observed or censored. +#' #' (\code{exact_indicator}): #' \itemize{ #' \item An \emph{observed} (exact) row has a known value, so it @@ -413,74 +390,70 @@ #' \code{L_i = (1 / sigma) phi(z_i)}. #' \item A \emph{censored} row is one whose intensity fell below the #' detection limit. Its true value is unknown; all we know is that it lies -#' somewhere below the threshold \code{c_i} (which -#' \code{.setCensoredByThreshold} has substituted in as \code{y_i}). The -#' honest contribution is therefore the total probability of landing -#' anywhere below that threshold - the normal CDF: +#' somewhere below the threshold \code{c_i}. #' \code{L_i = P(Y_i <= c_i) = Phi((c_i - mu_i) / sigma)}. #' } +#' #' Taking logs and summing over rows gives the objective that is maximized: #' \preformatted{ #' l(theta) = sum_{observed} [ log phi(z_i) - log sigma ] #' + sum_{censored} log Phi(z_i) #' } -#' (the first sum is, up to a constant, ordinary least squares; the second +#' +#' The first sum is, up to a constant, ordinary least squares; the second #' is what pulls \code{mu_i} and \code{sigma} toward values that make the -#' censored rows plausibly low). If a censored row's \code{mu_i} is well +#' censored rows plausibly low. If a censored row's \code{mu_i} is well #' above its threshold, \code{Phi(z_i)} is tiny and \code{l} is heavily -#' penalized - so the fit, and the imputed values later predicted from it, -#' respect the information that those rows were below the limit, rather -#' than ignoring them or treating the threshold as an exact value. -#' -#' \strong{Setting the derivatives to zero.} The maximum is where the score -#' (gradient of \code{l}) is zero. By the chain rule through -#' \code{mu_i = x_i' beta}, each row only needs its derivatives with respect -#' to \code{mu_i} and \code{log sigma} (computed by -#' \code{.aftGaussianDerivatives}): +#' penalized. +#' +#' @section Gradient ascent is performed to maximize the log likelihood. +#' +#' The maximum log likelihood is where the gradient is zero. We compute +#' derivatives with respect to each parameter: +#' #' \preformatted{ #' observed: d l_i / d mu_i = z_i / sigma #' censored: d l_i / d mu_i = -phi(z_i) / (sigma Phi(z_i)) #' } +#' #' The observed term is the usual least-squares residual pull; the censored -#' term (an inverse Mills ratio) always pushes \code{mu_i} down, strongly +#' term always pushes \code{mu_i} down, strongly #' when the prediction sits above the threshold and negligibly when it is -#' already well below. These assemble into the score -#' (\code{.buildAFTGradient}), with \code{d} the vector of -#' \code{d l_i / d mu_i}: -#' \preformatted{ -#' U(theta) = [ X' d ] (gradient wrt beta) -#' [ sum_i d l_i / d log sigma ] (gradient wrt log sigma) -#' } -#' There is no closed-form root because of the \code{Phi} terms, so the -#' root is found iteratively. +#' already well below. +#' +#' @section Step size is determined with the negative Hessian. +#' +#' Newton's method updates \code{theta} using the step that exactly +#' maximizes a second-order Taylor approximation of the +#' log-likelihood around the current estimate. #' -#' \strong{Newton-Raphson.} Expanding the score to first order around the -#' current guess, \code{U(theta + step) ~ U(theta) + H(theta) step}, and -#' setting it to zero gives the Newton step #' \preformatted{ -#' -H(theta) step = U(theta), theta_new = theta + step +#' l(theta) ~ l(theta_0) + g'(theta - theta_0) +#' + 1/2 (theta - theta_0)' H (theta - theta_0) #' } -#' where \code{H = d^2 l / d theta d theta'} is the Hessian of the -#' log-likelihood, with blocks +#' +#' Setting the derivative of this quadratic to zero and solving for +#' \code{theta} gives the update: +#' #' \preformatted{ -#' H = [ X' W X X' v ] W = diag(d^2 l_i / d mu_i^2) -#' [ v' X sum_i s_i ] v_i = d^2 l_i / d mu_i d log sigma -#' s_i = d^2 l_i / d (log sigma)^2 +#' theta_new = theta + (-H)^-1 * gradient #' } -#' The negative Hessian \code{-H} (\code{.buildAFTNegativeHessian}) is -#' also known as the observed information matrix. -#' This linear system is the part solved by \code{.cgSolve} (see its -#' documentation for the conjugate-gradient math). Starting values come -#' from ordinary least squares that ignores censoring. If a step fails to -#' increase \code{l} (or produces non-finite values), the candidate is -#' pulled toward the current guess, \code{(candidate + 2 * current) / 3}, -#' cutting the step to a third each time until \code{l} improves; if \code{-H} is not positive definite (so the -#' Newton direction might not point uphill), the Gauss-Newton matrix -#' \code{sum_i g_i g_i'} of per-row score contributions is used instead, -#' which is positive semi-definite by construction. Iteration stops when \code{l} changes by less than -#' \code{convergence_tolerance}, and the inverse of the final negative -#' Hessian is returned as the coefficient variance-covariance matrix, as -#' \code{survreg} does. +#' +#' Another way to think about this is that the Newton method rescales each +#' component of the gradient by an amount determined by local curvature, +#' rather than applying a single global step size. For example, if \code{-H} +#' were diagonal, this would reduce to an entry-specific +#' step size for each parameter: \code{theta_new_i = theta_i + +#' gradient_i / (-H_ii)}. Directions with sharp curvature (large +#' \code{|H_ii|}) get small steps, since the gradient there changes +#' quickly and is only locally reliable; directions with flat curvature +#' get large steps. +#' +#' In general \code{-H} is not diagonal, so \code{(-H)^-1} does not +#' just rescale each gradient entry independently. It captures how +#' curvature in one parameter's direction depends on the value of +#' another. This coupling is what makes Newton's method converge faster +#' than methods that rescale each coordinate independently. #' #' @param input data.table, the same shape \code{.fitSurvival} expects. #' @param aft_iterations maximum number of log-likelihood evaluations the @@ -494,22 +467,17 @@ #' @param use_jacobi_preconditioner if \code{TRUE}, precondition every #' conjugate-gradient solve with the inverse of the current negative #' Hessian's own diagonal (see \code{.cgSolve}'s -#' \code{use_jacobi_preconditioner}). This is what \code{aft_solver = -#' "pcg"} enables, versus plain conjugate gradient for \code{"cg"}. +#' \code{use_jacobi_preconditioner}). #' @param verbose if \code{TRUE}, \code{message()} a line per #' Newton-Raphson iteration - conjugate-gradient iterations used, whether #' the Gauss-Newton fallback (see below) was needed, elapsed time, and the #' resulting log-likelihood - plus a one-line summary once fitting -#' finishes. Meant for evaluating how solver choice and problem size -#' trade off against iteration count and wall time, not for routine use -#' (this fits one protein at a time, so it is easy to generate a line per -#' protein across a whole \code{dataProcess()} run). +#' finishes. #' #' @return a fitted model of class \code{"survreg"}, with one added field: #' \code{cg_diagnostics}, a data.frame with one row per Newton-Raphson #' iteration recording the conjugate-gradient iteration counts and timing -#' described above (populated regardless of \code{verbose}, so it can be -#' inspected/aggregated programmatically after the fact). +#' described above #' #' @importFrom stats model.frame model.matrix model.response lm.fit sd #' @keywords internal From a1ab4185d5e18818dfd96668b10a2b72ac7f31d1 Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Sun, 27 Sep 2026 13:53:01 -0400 Subject: [PATCH 30/30] update imputation comments --- R/utils_imputation.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/utils_imputation.R b/R/utils_imputation.R index 90745ddb..cf210ae2 100644 --- a/R/utils_imputation.R +++ b/R/utils_imputation.R @@ -371,7 +371,9 @@ } #' Fit a Gaussian, left-censored AFT model with a conjugate-gradient -#' Newton step (rather than a cholesky solve) +#' Newton step (rather than a cholesky solve). Maximum likelihood estimation +#' loop was written to match the survival package (survreg6.c) to ensure +#' results match with survreg. #' #' @section Under the hood, the AFT model is fit with maximum likelihood #' estimation, where the objective is a Gaussian density for observed rows and