diff --git a/NAMESPACE b/NAMESPACE index aa2226a..bda514a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,11 +1,13 @@ # Generated by roxygen2: do not edit by hand +S3method(print,topicHierarchy) export(annotateProteinInfoFromIndra) export(bootstrapTopicModels) export(compareTopicModels) export(cytoscapeNetwork) export(cytoscapeNetworkOutput) export(decomposeSubnetworkByTopic) +export(decomposeSubnetworkIntoHierarchicalTopics) export(deleteEdgeFromNetwork) export(exportNetworkToHTML) export(filterSubnetworkByContext) diff --git a/R/decomposeSubnetworkByTopic.R b/R/decomposeSubnetworkByTopic.R index ec24a3d..cd46e31 100644 --- a/R/decomposeSubnetworkByTopic.R +++ b/R/decomposeSubnetworkByTopic.R @@ -48,6 +48,14 @@ #' \code{FALSE}, NMF is run on the paper-word matrix only and edge-topic #' loadings are derived afterwards by folding edge counts onto the #' text-learned topics, so the PPIs do not influence the topics themselves. +#' @param evidence optional pre-fetched evidence data.frame, e.g. +#' \code{attr(topics, "corpus")$evidence} from a previous call. It is subset +#' to the edges of \code{subnetwork}, so the evidence gathered for a parent +#' network can be reused for any of its topic subnetworks. When \code{NULL} +#' (default) the evidence is queried from INDRA. +#' @param abstracts optional named character vector (or list) mapping PMID to +#' abstract text, e.g. \code{attr(topics, "corpus")$abstracts}. Only PMIDs +#' missing from it are fetched from PubMed. Default \code{NULL} fetches all. #' #' @return A list of length \code{n_topics}, named \code{topic_1} ... #' \code{topic_k}. Each element is a topic-specific subnetwork: a list with @@ -60,10 +68,14 @@ #' \item{pmids}{PMIDs whose strongest topic loading is this topic.} #' } #' The full factorization (W, H_text, H_edges, etc.) is attached as the -#' \code{"nmf"} attribute of the returned list. +#' \code{"nmf"} attribute of the returned list. The evidence and abstracts +#' used are attached as the \code{"corpus"} attribute (a list with +#' \code{evidence} and \code{abstracts}) so they can be passed back in via +#' the \code{evidence} and \code{abstracts} arguments. #' #' @seealso \code{\link{getSubnetworkFromIndra}}, -#' \code{\link{filterSubnetworkByContext}} +#' \code{\link{filterSubnetworkByContext}}, +#' \code{\link{decomposeSubnetworkIntoHierarchicalTopics}} #' #' @export #' @@ -80,6 +92,13 @@ #' topics <- decomposeSubnetworkByTopic(subnetwork, n_topics = 5) #' topics$topic_1$topTerms #' exportNetworkToHTML(topics$topic_1$nodes, topics$topic_1$edges) +#' +#' # Re-decompose a topic without re-querying INDRA / PubMed. +#' corpus <- attr(topics, "corpus") +#' topics_deeper <- decomposeSubnetworkByTopic( +#' topics$topic_1, n_topics = 5, +#' evidence = corpus$evidence, abstracts = corpus$abstracts +#' ) #' } decomposeSubnetworkByTopic <- function(subnetwork, n_topics = 5, @@ -89,13 +108,17 @@ decomposeSubnetworkByTopic <- function(subnetwork, max_iter = 200, tol = 1e-4, seed = 1, - include_ppi = TRUE) { + include_ppi = TRUE, + evidence = NULL, + abstracts = NULL) { .validateDecomposeSubnetworkByTopicInput(subnetwork, n_topics, edge_topic_cutoff, include_ppi) + .validateTopicCorpusInput(evidence, abstracts) # 1-3. Build the shared paper-by-word and paper-by-edge matrices. - mats <- .buildTopicMatrices(subnetwork, n_topics, min_term_count) + mats <- .buildTopicMatrices(subnetwork, n_topics, min_term_count, + evidence = evidence, abstracts = abstracts) nodes <- mats$nodes edges <- mats$edges pmids <- mats$pmids @@ -156,5 +179,9 @@ decomposeSubnetworkByTopic <- function(subnetwork, n_iter = model$n_iter, include_ppi = include_ppi ) + attr(topics, "corpus") <- list( + evidence = mats$evidence, + abstracts = mats$abstracts + ) return(topics) } diff --git a/R/decomposeSubnetworkIntoHierarchicalTopics.R b/R/decomposeSubnetworkIntoHierarchicalTopics.R new file mode 100644 index 0000000..06a0584 --- /dev/null +++ b/R/decomposeSubnetworkIntoHierarchicalTopics.R @@ -0,0 +1,359 @@ +#' Recursively decompose a subnetwork into a hierarchy of topic subnetworks +#' +#' Repeatedly applies \code{\link{decomposeSubnetworkByTopic}} to its own +#' topic subnetworks until every branch is small enough to inspect by hand +#' (at most \code{max_edges} edges), producing a topic tree: broad themes near +#' the root and increasingly specific sub-themes towards the leaves. +#' +#' INDRA evidence and PubMed abstracts are gathered once for the input +#' subnetwork and reused for every sub-decomposition, so no further network +#' requests are made during the recursion. Each sub-decomposition rebuilds its +#' vocabulary and refits the NMF on only the papers supporting that branch's +#' edges, which lets finer topics emerge. +#' +#' A branch stops splitting (becomes a leaf) when any of the following holds, +#' recorded in the \code{stop_reason} column of \code{tree}: +#' \describe{ +#' \item{small_enough}{it has at most \code{max_edges} edges.} +#' \item{max_depth}{it sits at depth \code{max_depth}.} +#' \item{too_few_papers}{fewer than two papers support its edges.} +#' \item{no_split}{every child topic contained all of its edges (or none), +#' so splitting would make no progress.} +#' \item{failed: }{the decomposition raised an error, e.g. no +#' usable words in the branch's abstracts.} +#' } +#' Edges without any PMID-backed evidence cannot be assigned to a topic and +#' only appear at the root. +#' +#' @param subnetwork list with \code{nodes} and \code{edges} data.frames, e.g. +#' the output of \code{\link{getSubnetworkFromIndra}}. +#' @param max_edges a branch with at most this many edges is not split +#' further. Default 10. +#' @param n_topics number of topics per split. Default 5. +#' @param edge_topic_cutoff topic-share threshold for assigning an edge to a +#' topic at each split; see \code{\link{decomposeSubnetworkByTopic}}. The +#' default of 0.9 gives a near-partition, so sibling topics rarely share +#' edges. Lower values allow an edge to appear in several sibling topics. +#' @param max_depth maximum depth of the tree (the root is depth 0). Default 5. +#' @param evidence optional pre-fetched evidence data.frame, e.g. +#' \code{attr(topics, "corpus")$evidence} from +#' \code{\link{decomposeSubnetworkByTopic}} or \code{result$corpus$evidence} +#' from a previous call of this function. Default \code{NULL} queries INDRA +#' once. +#' @param abstracts optional named character vector mapping PMID to abstract +#' text. Only missing PMIDs are fetched from PubMed. Default \code{NULL}. +#' @param ... further arguments passed to +#' \code{\link{decomposeSubnetworkByTopic}}, e.g. \code{n_top_terms}, +#' \code{min_term_count}, \code{include_ppi}, \code{seed}. +#' +#' @return An object of class \code{topicHierarchy}: a list with +#' \describe{ +#' \item{tree}{data.frame with one row per topic, in depth-first order. +#' Columns: \code{id} (\code{"root"}, \code{"1"}, \code{"1.2"}, ...), +#' \code{parent_id}, \code{depth}, \code{topic} (index within the +#' parent's split), \code{n_edges}, \code{n_nodes}, \code{n_papers} +#' (papers supporting the topic's edges), \code{n_children}, +#' \code{is_leaf}, \code{stop_reason}, \code{mean_topic_weight} (mean +#' share of the topic's edges' loading, a cohesion score), +#' \code{top_terms} (collapsed with \code{", "}), \code{label}, and +#' \code{pathString} (\code{"root/1/1.2"}, for \code{data.tree}).} +#' \item{subnetworks}{named list keyed by \code{id}; each element is a +#' subnetwork (\code{nodes}, \code{edges}, \code{topTerms}, +#' \code{pmids}) that can be passed to \code{\link{cytoscapeNetwork}} or +#' \code{\link{exportNetworkToHTML}}.} +#' \item{edge_membership}{long data.frame with one row per (topic, edge): +#' \code{id}, \code{depth}, \code{is_leaf}, \code{source}, +#' \code{target}, \code{interaction}, \code{topicWeight}. Filter on +#' \code{is_leaf} to see which fine-grained topic(s) each edge ends up +#' in.} +#' \item{corpus}{list with the \code{evidence} and \code{abstracts} used, +#' for reuse via the \code{evidence} and \code{abstracts} arguments.} +#' \item{params}{the settings used.} +#' } +#' +#' @seealso \code{\link{decomposeSubnetworkByTopic}} +#' @note \strong{Beta feature:} This function is experimental and the API may +#' change without notice in future versions. +#' @export +#' +#' @examples +#' \dontrun{ +#' input <- data.table::fread(system.file( +#' "extdata/groupComparisonModel.csv", +#' package = "MSstatsBioNet" +#' )) +#' subnetwork <- getSubnetworkFromIndra(input) +#' hierarchy <- decomposeSubnetworkIntoHierarchicalTopics( +#' subnetwork, max_edges = 10, n_topics = 5, edge_topic_cutoff = 0.9 +#' ) +#' hierarchy # indented topic tree +#' leaves <- hierarchy$tree[hierarchy$tree$is_leaf, ] +#' leaves[order(-leaves$mean_topic_weight), c("id", "n_edges", "top_terms")] +#' +#' # Inspect one fine-grained topic as a network. +#' leaf <- hierarchy$subnetworks[["1.2"]] +#' exportNetworkToHTML(leaf$nodes, leaf$edges) +#' +#' # Tree visualization with other packages, e.g. +#' # data.tree::as.Node(hierarchy$tree) +#' # igraph::graph_from_data_frame( +#' # hierarchy$tree[-1, c("parent_id", "id")]) +#' } +decomposeSubnetworkIntoHierarchicalTopics <- function(subnetwork, + max_edges = 10, + n_topics = 5, + edge_topic_cutoff = 0.9, + max_depth = 5, + evidence = NULL, + abstracts = NULL, + ...) { + + .validateDecomposeSubnetworkByTopicInput(subnetwork, n_topics, + edge_topic_cutoff) + .validateTopicCorpusInput(evidence, abstracts) + if (!is.numeric(max_edges) || length(max_edges) != 1L || + is.na(max_edges) || max_edges < 1) { + stop("`max_edges` must be a single number >= 1.") + } + if (!is.numeric(max_depth) || length(max_depth) != 1L || + is.na(max_depth) || max_depth < 0 || + max_depth != as.integer(max_depth)) { + stop("`max_depth` must be a single non-negative integer.") + } + + # Gather the corpus once; every sub-decomposition reuses it. + corpus <- .gatherTopicCorpus(subnetwork$edges, evidence, abstracts) + + root <- list(nodes = subnetwork$nodes, edges = subnetwork$edges, + topTerms = character(0), pmids = character(0)) + stack <- list(list(id = "root", parent_id = NA_character_, depth = 0L, + topic = NA_integer_, subnetwork = root)) + rows <- list() + subnetworks <- list() + + # Depth-first traversal so rows come out in tree (print) order. + while (length(stack) > 0) { + current <- stack[[length(stack)]] + stack[[length(stack)]] <- NULL + sub <- current$subnetwork + + split <- .splitTopicNode(sub, current$depth, corpus, max_edges, + max_depth, n_topics, edge_topic_cutoff, ...) + children <- split$children + child_ids <- paste0(if (current$id == "root") "" else + paste0(current$id, "."), + names(children)) + for (i in rev(seq_along(children))) { + stack[[length(stack) + 1]] <- list( + id = child_ids[i], parent_id = current$id, + depth = current$depth + 1L, topic = children[[i]]$topic, + subnetwork = children[[i]] + ) + } + + rows[[length(rows) + 1]] <- .topicNodeRow(current, split, + length(children)) + subnetworks[[current$id]] <- sub[c("nodes", "edges", + "topTerms", "pmids")] + } + + tree <- do.call(rbind, rows) + rownames(tree) <- NULL + tree$pathString <- .topicPathStrings(tree$id, tree$parent_id) + + result <- list( + tree = tree, + subnetworks = subnetworks, + edge_membership = .topicEdgeMembership(tree, subnetworks), + corpus = corpus, + params = list(max_edges = max_edges, n_topics = n_topics, + edge_topic_cutoff = edge_topic_cutoff, + max_depth = max_depth, ...) + ) + class(result) <- "topicHierarchy" + result +} + + +#' Print a topic hierarchy as an indented tree +#' +#' @param x a \code{topicHierarchy} object from +#' \code{\link{decomposeSubnetworkIntoHierarchicalTopics}}. +#' @param n_terms number of top terms to show per topic. Default 5. +#' @param ... ignored. +#' @return \code{x}, invisibly. +#' @export +print.topicHierarchy <- function(x, n_terms = 5, ...) { + tree <- x$tree + cat(sprintf( + "Topic hierarchy: %d topics, %d leaves, depth %d (max_edges = %s)\n", + nrow(tree) - 1L, sum(tree$is_leaf & tree$depth > 0), + max(tree$depth), format(x$params$max_edges) + )) + for (i in seq_len(nrow(tree))) { + terms <- x$subnetworks[[tree$id[i]]]$topTerms + terms <- paste(utils::head(terms, n_terms), collapse = ", ") + stop_note <- if (tree$is_leaf[i] && tree$depth[i] > 0 && + tree$stop_reason[i] != "small_enough") { + paste0(" <", tree$stop_reason[i], ">") + } else "" + cat(sprintf("%s%s [%d edges, %d papers]%s%s\n", + strrep(" ", tree$depth[i]), tree$id[i], + tree$n_edges[i], tree$n_papers[i], + if (nzchar(terms)) paste0(" ", terms) else "", + stop_note)) + } + invisible(x) +} + + +#' Decide whether a topic node should split, and split it if so +#' +#' @param sub topic subnetwork (list with `nodes` and `edges`) +#' @param depth depth of the node in the tree +#' @param corpus list with `evidence` and `abstracts` from +#' \code{.gatherTopicCorpus} +#' @param max_edges,max_depth,n_topics,edge_topic_cutoff see +#' \code{decomposeSubnetworkIntoHierarchicalTopics} +#' @param ... passed to \code{decomposeSubnetworkByTopic} +#' @return list with `children` (named list of child topic subnetworks, empty +#' for a leaf), `stop_reason` (NA when the node split), and `n_papers` +#' @keywords internal +#' @noRd +.splitTopicNode <- function(sub, depth, corpus, max_edges, max_depth, + n_topics, edge_topic_cutoff, ...) { + n_edges <- nrow(sub$edges) + node_evidence <- .subsetEvidenceToEdges(corpus$evidence, sub$edges) + n_papers <- length(unique(node_evidence$pmid)) + leaf <- function(reason) { + list(children = list(), stop_reason = reason, n_papers = n_papers) + } + + if (n_edges <= max_edges) return(leaf("small_enough")) + if (depth >= max_depth) return(leaf("max_depth")) + if (n_papers < 2) return(leaf("too_few_papers")) + + topics <- tryCatch( + withCallingHandlers( + decomposeSubnetworkByTopic( + sub, n_topics = n_topics, + edge_topic_cutoff = edge_topic_cutoff, + evidence = node_evidence, abstracts = corpus$abstracts, ... + ), + # n_topics is routinely capped by the paper count deep in the tree. + warning = function(w) { + if (grepl("^Only \\d+ papers available", + conditionMessage(w))) { + invokeRestart("muffleWarning") + } + } + ), + error = function(e) e + ) + if (inherits(topics, "error")) { + return(leaf(paste0("failed: ", conditionMessage(topics)))) + } + + # Keep only children that strictly shrink the branch; this also + # guarantees the recursion terminates. + n_child_edges <- vapply(topics, function(t) nrow(t$edges), integer(1)) + children <- topics[n_child_edges > 0 & n_child_edges < n_edges] + if (length(children) == 0) return(leaf("no_split")) + names(children) <- vapply(children, function(t) as.character(t$topic), + character(1)) + + list(children = children, stop_reason = NA_character_, + n_papers = n_papers) +} + + +#' Build the summary row of the tree data.frame for one topic node +#' @param current stack entry (id, parent_id, depth, topic, subnetwork) +#' @param split output of \code{.splitTopicNode} +#' @param n_children number of children the node was split into +#' @return one-row data.frame +#' @keywords internal +#' @noRd +.topicNodeRow <- function(current, split, n_children) { + sub <- current$subnetwork + weights <- sub$edges$topicWeight + top_terms <- paste(sub$topTerms, collapse = ", ") + data.frame( + id = current$id, + parent_id = current$parent_id, + depth = current$depth, + topic = current$topic, + n_edges = nrow(sub$edges), + n_nodes = nrow(sub$nodes), + n_papers = split$n_papers, + n_children = n_children, + is_leaf = n_children == 0, + stop_reason = split$stop_reason, + mean_topic_weight = if (is.null(weights) || current$id == "root") + NA_real_ else mean(weights), + top_terms = top_terms, + label = sprintf("%s (%d edges)%s", current$id, + nrow(sub$edges), + if (nzchar(top_terms)) + paste0(": ", paste(utils::head( + sub$topTerms, 3), + collapse = ", ")) + else ""), + stringsAsFactors = FALSE + ) +} + + +#' Build data.tree-style path strings ("root/1/1.2") for each tree node +#' @param ids character vector of node ids +#' @param parent_ids character vector of parent ids (NA for the root) +#' @return character vector of path strings aligned to `ids` +#' @keywords internal +#' @noRd +.topicPathStrings <- function(ids, parent_ids) { + parent_of <- stats::setNames(parent_ids, ids) + vapply(ids, function(id) { + path <- id + while (!is.na(parent_of[[id]])) { + id <- parent_of[[id]] + path <- c(id, path) + } + paste(path, collapse = "/") + }, character(1), USE.NAMES = FALSE) +} + + +#' Long table of topic-to-edge membership across the hierarchy +#' @param tree tree data.frame +#' @param subnetworks named list of topic subnetworks keyed by tree id +#' @return data.frame with one row per (non-root topic, edge) +#' @keywords internal +#' @noRd +.topicEdgeMembership <- function(tree, subnetworks) { + non_root <- tree[tree$id != "root", , drop = FALSE] + parts <- lapply(seq_len(nrow(non_root)), function(i) { + edges <- subnetworks[[non_root$id[i]]]$edges + if (nrow(edges) == 0) return(NULL) + data.frame( + id = non_root$id[i], + depth = non_root$depth[i], + is_leaf = non_root$is_leaf[i], + source = edges$source, + target = edges$target, + interaction = edges$interaction, + topicWeight = edges$topicWeight, + stringsAsFactors = FALSE + ) + }) + parts <- parts[!vapply(parts, is.null, logical(1))] + if (length(parts) == 0) { + return(data.frame(id = character(), depth = integer(), + is_leaf = logical(), source = character(), + target = character(), interaction = character(), + topicWeight = numeric(), + stringsAsFactors = FALSE)) + } + out <- do.call(rbind, parts) + rownames(out) <- NULL + out +} diff --git a/R/utils_decomposeSubnetworkByTopic.R b/R/utils_decomposeSubnetworkByTopic.R index b783faa..6c3189c 100644 --- a/R/utils_decomposeSubnetworkByTopic.R +++ b/R/utils_decomposeSubnetworkByTopic.R @@ -68,18 +68,26 @@ #' @param n_topics requested number of topics; reduced (with a warning) when #' fewer papers than topics are available #' @param min_term_count minimum corpus term frequency to keep a word -#' @return list with `nodes`, `edges`, `evidence`, `pmids`, `edge_keys`, -#' `X_text`, `X_edges`, and the (possibly reduced) `n_topics` +#' @param evidence optional pre-fetched evidence data.frame (as returned by +#' \code{.extract_evidence_text}); subset to the subnetwork's edges. When +#' NULL, evidence is queried from INDRA. +#' @param abstracts optional named character vector (or list) mapping PMID to +#' abstract text. Only PMIDs missing from it are fetched from PubMed. +#' @return list with `nodes`, `edges`, `evidence`, `abstracts`, `pmids`, +#' `edge_keys`, `X_text`, `X_edges`, and the (possibly reduced) `n_topics` #' @keywords internal #' @noRd -.buildTopicMatrices <- function(subnetwork, n_topics, min_term_count = 2) { +.buildTopicMatrices <- function(subnetwork, n_topics, min_term_count = 2, + evidence = NULL, abstracts = NULL) { nodes <- subnetwork$nodes edges <- subnetwork$edges n_topics <- as.integer(n_topics) - # 1. Evidence (paper <-> edge links) for every edge. - evidence <- .extract_evidence_text(edges) - evidence <- evidence[!is.na(evidence$pmid) & nchar(evidence$pmid) > 0, ] + # 1. Evidence (paper <-> edge links) for every edge and the abstracts of + # the supporting papers, reusing any supplied evidence / abstracts. + corpus <- .gatherTopicCorpus(edges, evidence, abstracts) + evidence <- corpus$evidence + abstracts <- corpus$abstracts if (nrow(evidence) == 0) { stop("No evidence with PMIDs was found for any edge; ", "cannot decompose into topics.") @@ -98,22 +106,110 @@ } # 2. X_text (papers x words) from PubMed abstracts. - abstract_list <- .fetch_clean_abstracts_xml(pmids) - abstracts <- vapply(pmids, function(p) { - a <- abstract_list[[p]] - if (is.null(a)) "" else a - }, character(1)) X_text <- .buildTextMatrix(pmids, abstracts, min_term_count) # 3. X_edges (papers x source_target_interaction) of evidence counts. X_edges <- .buildEdgeMatrix(evidence, pmids, edge_keys) list(nodes = nodes, edges = edges, evidence = evidence, - pmids = pmids, edge_keys = edge_keys, + abstracts = abstracts, pmids = pmids, edge_keys = edge_keys, X_text = X_text, X_edges = X_edges, n_topics = n_topics) } +#' Gather the INDRA evidence and PubMed abstracts for a set of edges +#' +#' Network-bound step shared by \code{\link{decomposeSubnetworkByTopic}} and +#' \code{\link{decomposeSubnetworkIntoHierarchicalTopics}}. A supplied +#' `evidence` table is subset to `edges` instead of re-querying INDRA, and only +#' PMIDs missing from `abstracts` are fetched from PubMed. +#' +#' @param edges edges data.frame +#' @param evidence NULL or pre-fetched evidence data.frame +#' @param abstracts NULL or named character vector / list of abstracts +#' @return list with `evidence` (PMID-backed rows only) and `abstracts` +#' (named character vector covering every PMID in `evidence`) +#' @keywords internal +#' @noRd +.gatherTopicCorpus <- function(edges, evidence = NULL, abstracts = NULL) { + if (is.null(evidence)) { + evidence <- .extract_evidence_text(edges) + } else { + evidence <- .subsetEvidenceToEdges(evidence, edges) + } + evidence <- evidence[!is.na(evidence$pmid) & nchar(evidence$pmid) > 0, , + drop = FALSE] + pmids <- unique(evidence$pmid) + + abstract_list <- if (is.null(abstracts)) list() else as.list(abstracts) + missing_pmids <- setdiff(pmids, names(abstract_list)) + if (length(missing_pmids) > 0) { + abstract_list <- c(abstract_list, + .fetch_clean_abstracts_xml(missing_pmids)) + } + abstracts <- vapply(pmids, function(p) { + a <- abstract_list[[p]] + if (is.null(a) || is.na(a)) "" else as.character(a) + }, character(1)) + + list(evidence = evidence, abstracts = abstracts) +} + + +#' Restrict a pre-fetched evidence table to the edges of a subnetwork +#' +#' Matches on the edge key (source, target, interaction) together with the +#' statement hash, so the result equals what \code{.extract_evidence_text} +#' would return for \code{edges} without re-querying INDRA. +#' +#' @param evidence evidence data.frame with `source`, `target`, `interaction`, +#' `stmt_hash`, and `pmid` columns +#' @param edges edges data.frame with `source`, `target`, `interaction`, and +#' `stmt_hash` columns +#' @return subset of `evidence` +#' @keywords internal +#' @noRd +.subsetEvidenceToEdges <- function(evidence, edges) { + ev_id <- paste(.edgeKey(evidence$source, evidence$target, + evidence$interaction), + as.character(evidence$stmt_hash), sep = "##") + edge_id <- paste(.edgeKey(edges$source, edges$target, edges$interaction), + as.character(edges$stmt_hash), sep = "##") + evidence[ev_id %in% edge_id, , drop = FALSE] +} + + +#' Validate optional pre-fetched evidence / abstracts inputs +#' @param evidence NULL or evidence data.frame +#' @param abstracts NULL or named character vector / list of abstracts +#' @keywords internal +#' @noRd +.validateTopicCorpusInput <- function(evidence, abstracts) { + if (!is.null(evidence)) { + required_cols <- c("source", "target", "interaction", + "stmt_hash", "pmid") + if (!is.data.frame(evidence) || + !all(required_cols %in% names(evidence))) { + stop("`evidence` must be a data.frame with columns: ", + paste(required_cols, collapse = ", "), + ", e.g. attr(decomposeSubnetworkByTopic(...), ", + "\"corpus\")$evidence.") + } + } + if (!is.null(abstracts)) { + if (!(is.character(abstracts) || is.list(abstracts)) || + (length(abstracts) > 0 && is.null(names(abstracts))) || + (is.list(abstracts) && + any(!vapply(abstracts, function(a) { + is.character(a) && length(a) == 1L + }, logical(1))))) { + stop("`abstracts` must be a named character vector or list ", + "mapping PMID to abstract text.") + } + } +} + + #' Build the paper-by-word matrix (X_text) from PubMed abstracts #' #' Tokenises abstracts with text2vec and returns a dense paper-by-word count diff --git a/man/decomposeSubnetworkByTopic.Rd b/man/decomposeSubnetworkByTopic.Rd index e29afd7..8db14b0 100644 --- a/man/decomposeSubnetworkByTopic.Rd +++ b/man/decomposeSubnetworkByTopic.Rd @@ -13,7 +13,9 @@ decomposeSubnetworkByTopic( max_iter = 200, tol = 1e-04, seed = 1, - include_ppi = TRUE + include_ppi = TRUE, + evidence = NULL, + abstracts = NULL ) } \arguments{ @@ -44,6 +46,16 @@ factorized jointly with the text matrix via a shared basis. If \code{FALSE}, NMF is run on the paper-word matrix only and edge-topic loadings are derived afterwards by folding edge counts onto the text-learned topics, so the PPIs do not influence the topics themselves.} + +\item{evidence}{optional pre-fetched evidence data.frame, e.g. +\code{attr(topics, "corpus")$evidence} from a previous call. It is subset +to the edges of \code{subnetwork}, so the evidence gathered for a parent +network can be reused for any of its topic subnetworks. When \code{NULL} +(default) the evidence is queried from INDRA.} + +\item{abstracts}{optional named character vector (or list) mapping PMID to +abstract text, e.g. \code{attr(topics, "corpus")$abstracts}. Only PMIDs +missing from it are fetched from PubMed. Default \code{NULL} fetches all.} } \value{ A list of length \code{n_topics}, named \code{topic_1} ... @@ -57,7 +69,10 @@ A list of length \code{n_topics}, named \code{topic_1} ... \item{pmids}{PMIDs whose strongest topic loading is this topic.} } The full factorization (W, H_text, H_edges, etc.) is attached as the - \code{"nmf"} attribute of the returned list. + \code{"nmf"} attribute of the returned list. The evidence and abstracts + used are attached as the \code{"corpus"} attribute (a list with + \code{evidence} and \code{abstracts}) so they can be passed back in via + the \code{evidence} and \code{abstracts} arguments. } \description{ Takes a subnetwork (the output of \code{\link{getSubnetworkFromIndra}}) and @@ -104,9 +119,17 @@ subnetwork <- getSubnetworkFromIndra(input) topics <- decomposeSubnetworkByTopic(subnetwork, n_topics = 5) topics$topic_1$topTerms exportNetworkToHTML(topics$topic_1$nodes, topics$topic_1$edges) + +# Re-decompose a topic without re-querying INDRA / PubMed. +corpus <- attr(topics, "corpus") +topics_deeper <- decomposeSubnetworkByTopic( + topics$topic_1, n_topics = 5, + evidence = corpus$evidence, abstracts = corpus$abstracts +) } } \seealso{ \code{\link{getSubnetworkFromIndra}}, - \code{\link{filterSubnetworkByContext}} + \code{\link{filterSubnetworkByContext}}, + \code{\link{decomposeSubnetworkIntoHierarchicalTopics}} } diff --git a/man/decomposeSubnetworkIntoHierarchicalTopics.Rd b/man/decomposeSubnetworkIntoHierarchicalTopics.Rd new file mode 100644 index 0000000..d7b3d16 --- /dev/null +++ b/man/decomposeSubnetworkIntoHierarchicalTopics.Rd @@ -0,0 +1,130 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/decomposeSubnetworkIntoHierarchicalTopics.R +\name{decomposeSubnetworkIntoHierarchicalTopics} +\alias{decomposeSubnetworkIntoHierarchicalTopics} +\title{Recursively decompose a subnetwork into a hierarchy of topic subnetworks} +\usage{ +decomposeSubnetworkIntoHierarchicalTopics( + subnetwork, + max_edges = 10, + n_topics = 5, + edge_topic_cutoff = 0.9, + max_depth = 5, + evidence = NULL, + abstracts = NULL, + ... +) +} +\arguments{ +\item{subnetwork}{list with \code{nodes} and \code{edges} data.frames, e.g. +the output of \code{\link{getSubnetworkFromIndra}}.} + +\item{max_edges}{a branch with at most this many edges is not split +further. Default 10.} + +\item{n_topics}{number of topics per split. Default 5.} + +\item{edge_topic_cutoff}{topic-share threshold for assigning an edge to a +topic at each split; see \code{\link{decomposeSubnetworkByTopic}}. The +default of 0.9 gives a near-partition, so sibling topics rarely share +edges. Lower values allow an edge to appear in several sibling topics.} + +\item{max_depth}{maximum depth of the tree (the root is depth 0). Default 5.} + +\item{evidence}{optional pre-fetched evidence data.frame, e.g. +\code{attr(topics, "corpus")$evidence} from +\code{\link{decomposeSubnetworkByTopic}} or \code{result$corpus$evidence} +from a previous call of this function. Default \code{NULL} queries INDRA +once.} + +\item{abstracts}{optional named character vector mapping PMID to abstract +text. Only missing PMIDs are fetched from PubMed. Default \code{NULL}.} + +\item{...}{further arguments passed to +\code{\link{decomposeSubnetworkByTopic}}, e.g. \code{n_top_terms}, +\code{min_term_count}, \code{include_ppi}, \code{seed}.} +} +\value{ +An object of class \code{topicHierarchy}: a list with + \describe{ + \item{tree}{data.frame with one row per topic, in depth-first order. + Columns: \code{id} (\code{"root"}, \code{"1"}, \code{"1.2"}, ...), + \code{parent_id}, \code{depth}, \code{topic} (index within the + parent's split), \code{n_edges}, \code{n_nodes}, \code{n_papers} + (papers supporting the topic's edges), \code{n_children}, + \code{is_leaf}, \code{stop_reason}, \code{mean_topic_weight} (mean + share of the topic's edges' loading, a cohesion score), + \code{top_terms} (collapsed with \code{", "}), \code{label}, and + \code{pathString} (\code{"root/1/1.2"}, for \code{data.tree}).} + \item{subnetworks}{named list keyed by \code{id}; each element is a + subnetwork (\code{nodes}, \code{edges}, \code{topTerms}, + \code{pmids}) that can be passed to \code{\link{cytoscapeNetwork}} or + \code{\link{exportNetworkToHTML}}.} + \item{edge_membership}{long data.frame with one row per (topic, edge): + \code{id}, \code{depth}, \code{is_leaf}, \code{source}, + \code{target}, \code{interaction}, \code{topicWeight}. Filter on + \code{is_leaf} to see which fine-grained topic(s) each edge ends up + in.} + \item{corpus}{list with the \code{evidence} and \code{abstracts} used, + for reuse via the \code{evidence} and \code{abstracts} arguments.} + \item{params}{the settings used.} + } +} +\description{ +Repeatedly applies \code{\link{decomposeSubnetworkByTopic}} to its own +topic subnetworks until every branch is small enough to inspect by hand +(at most \code{max_edges} edges), producing a topic tree: broad themes near +the root and increasingly specific sub-themes towards the leaves. +} +\details{ +INDRA evidence and PubMed abstracts are gathered once for the input +subnetwork and reused for every sub-decomposition, so no further network +requests are made during the recursion. Each sub-decomposition rebuilds its +vocabulary and refits the NMF on only the papers supporting that branch's +edges, which lets finer topics emerge. + +A branch stops splitting (becomes a leaf) when any of the following holds, +recorded in the \code{stop_reason} column of \code{tree}: +\describe{ + \item{small_enough}{it has at most \code{max_edges} edges.} + \item{max_depth}{it sits at depth \code{max_depth}.} + \item{too_few_papers}{fewer than two papers support its edges.} + \item{no_split}{every child topic contained all of its edges (or none), + so splitting would make no progress.} + \item{failed: }{the decomposition raised an error, e.g. no + usable words in the branch's abstracts.} +} +Edges without any PMID-backed evidence cannot be assigned to a topic and +only appear at the root. +} +\note{ +\strong{Beta feature:} This function is experimental and the API may + change without notice in future versions. +} +\examples{ +\dontrun{ +input <- data.table::fread(system.file( + "extdata/groupComparisonModel.csv", + package = "MSstatsBioNet" +)) +subnetwork <- getSubnetworkFromIndra(input) +hierarchy <- decomposeSubnetworkIntoHierarchicalTopics( + subnetwork, max_edges = 10, n_topics = 5, edge_topic_cutoff = 0.9 +) +hierarchy # indented topic tree +leaves <- hierarchy$tree[hierarchy$tree$is_leaf, ] +leaves[order(-leaves$mean_topic_weight), c("id", "n_edges", "top_terms")] + +# Inspect one fine-grained topic as a network. +leaf <- hierarchy$subnetworks[["1.2"]] +exportNetworkToHTML(leaf$nodes, leaf$edges) + +# Tree visualization with other packages, e.g. +# data.tree::as.Node(hierarchy$tree) +# igraph::graph_from_data_frame( +# hierarchy$tree[-1, c("parent_id", "id")]) +} +} +\seealso{ +\code{\link{decomposeSubnetworkByTopic}} +} diff --git a/man/print.topicHierarchy.Rd b/man/print.topicHierarchy.Rd new file mode 100644 index 0000000..472d903 --- /dev/null +++ b/man/print.topicHierarchy.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/decomposeSubnetworkIntoHierarchicalTopics.R +\name{print.topicHierarchy} +\alias{print.topicHierarchy} +\title{Print a topic hierarchy as an indented tree} +\usage{ +\method{print}{topicHierarchy}(x, n_terms = 5, ...) +} +\arguments{ +\item{x}{a \code{topicHierarchy} object from +\code{\link{decomposeSubnetworkIntoHierarchicalTopics}}.} + +\item{n_terms}{number of top terms to show per topic. Default 5.} + +\item{...}{ignored.} +} +\value{ +\code{x}, invisibly. +} +\description{ +Print a topic hierarchy as an indented tree +} diff --git a/tests/testthat/test-decomposeSubnetworkIntoHierarchicalTopics.R b/tests/testthat/test-decomposeSubnetworkIntoHierarchicalTopics.R new file mode 100644 index 0000000..7175762 --- /dev/null +++ b/tests/testthat/test-decomposeSubnetworkIntoHierarchicalTopics.R @@ -0,0 +1,239 @@ +# Synthetic corpus: three themes, each with its own vocabulary, papers, and +# edges, so the NMF has clear structure to recover without any network calls. +THEME_WORDS <- list( + c("kinase", "phosphorylation", "signaling", "cascade", "mapk", "erk"), + c("dna", "repair", "damage", "checkpoint", "replication", "genome"), + c("immune", "cytokine", "inflammation", "macrophage", "tcell", "interferon") +) + +make_theme_corpus <- function(edges_per_theme = 15, papers_per_theme = 8) { + edges <- list() + evidence <- list() + abstracts <- character(0) + for (th in seq_along(THEME_WORDS)) { + pmids <- paste0("PM", th, "_", seq_len(papers_per_theme)) + words <- THEME_WORDS[[th]] + abstracts[pmids] <- vapply(seq_along(pmids), function(i) { + paste(rep(words, times = 3 + (i %% 3)), collapse = " ") + }, character(1)) + for (e in seq_len(edges_per_theme)) { + src <- paste0("G", th, "_", e) + tgt <- paste0("G", th, "_", e + 1) + hash <- paste0("h", th, "_", e) + edges[[length(edges) + 1]] <- data.frame( + source = src, target = tgt, interaction = "Activation", + site = NA_character_, evidenceLink = "https://example.com", + stmt_hash = hash, stringsAsFactors = FALSE + ) + ev_pmids <- pmids[c(e %% papers_per_theme + 1, + (e + 3) %% papers_per_theme + 1)] + evidence[[length(evidence) + 1]] <- data.frame( + source = src, target = tgt, interaction = "Activation", + site = NA_character_, evidenceLink = "https://example.com", + stmt_hash = hash, text = "sentence", pmid = ev_pmids, + stringsAsFactors = FALSE + ) + } + } + edges <- do.call(rbind, edges) + nodes <- data.frame(id = unique(c(edges$source, edges$target)), + stringsAsFactors = FALSE) + list(subnetwork = list(nodes = nodes, edges = edges), + evidence = do.call(rbind, evidence), + abstracts = abstracts) +} + +forbid_network <- function(env = parent.frame()) { + testthat::local_mocked_bindings( + .extract_evidence_text = function(...) stop("INDRA was queried"), + .fetch_clean_abstracts_xml = function(...) stop("PubMed was queried"), + .env = env + ) +} + +describe("decomposeSubnetworkByTopic with a supplied corpus", { + + test_that("uses the supplied evidence and abstracts without fetching", { + forbid_network() + corpus <- make_theme_corpus() + topics <- decomposeSubnetworkByTopic( + corpus$subnetwork, n_topics = 3, + evidence = corpus$evidence, abstracts = corpus$abstracts + ) + expect_length(topics, 3) + used <- attr(topics, "corpus") + expect_setequal(names(used$abstracts), unique(corpus$evidence$pmid)) + expect_equal(nrow(used$evidence), nrow(corpus$evidence)) + }) + + test_that("a topic subnetwork can be re-decomposed from the corpus attr", { + forbid_network() + corpus <- make_theme_corpus() + topics <- decomposeSubnetworkByTopic( + corpus$subnetwork, n_topics = 3, edge_topic_cutoff = 0.9, + evidence = corpus$evidence, abstracts = corpus$abstracts + ) + used <- attr(topics, "corpus") + deeper <- decomposeSubnetworkByTopic( + topics$topic_1, n_topics = 2, + evidence = used$evidence, abstracts = used$abstracts + ) + # Evidence is restricted to the parent topic's edges. + sub_used <- attr(deeper, "corpus")$evidence + expect_true(all(sub_used$stmt_hash %in% topics$topic_1$edges$stmt_hash)) + }) + + test_that("only PMIDs missing from `abstracts` are fetched", { + corpus <- make_theme_corpus() + missing <- names(corpus$abstracts)[1:2] + fetched <- NULL + testthat::local_mocked_bindings( + .fetch_clean_abstracts_xml = function(pmids, ...) { + fetched <<- pmids + as.list(stats::setNames(corpus$abstracts[pmids], pmids)) + } + ) + decomposeSubnetworkByTopic( + corpus$subnetwork, n_topics = 3, evidence = corpus$evidence, + abstracts = corpus$abstracts[-(1:2)] + ) + expect_setequal(fetched, missing) + }) + + test_that("rejects malformed evidence and abstracts", { + corpus <- make_theme_corpus() + expect_error( + decomposeSubnetworkByTopic(corpus$subnetwork, + evidence = data.frame(x = 1)), + "`evidence` must be a data.frame" + ) + expect_error( + decomposeSubnetworkByTopic(corpus$subnetwork, + evidence = corpus$evidence, + abstracts = unname(corpus$abstracts)), + "`abstracts` must be a named" + ) + }) +}) + +describe("decomposeSubnetworkIntoHierarchicalTopics", { + + test_that("splits until every leaf is small or cannot split further", { + forbid_network() + corpus <- make_theme_corpus() + h <- decomposeSubnetworkIntoHierarchicalTopics( + corpus$subnetwork, max_edges = 10, n_topics = 3, + evidence = corpus$evidence, abstracts = corpus$abstracts + ) + expect_s3_class(h, "topicHierarchy") + tree <- h$tree + expect_equal(tree$id[1], "root") + expect_true(is.na(tree$parent_id[1])) + expect_equal(tree$n_edges[1], nrow(corpus$subnetwork$edges)) + expect_true(any(tree$depth > 0)) + + leaves <- tree[tree$is_leaf, ] + expect_true(all(leaves$n_edges <= 10 | + leaves$stop_reason != "small_enough")) + expect_true(all(is.na(tree$stop_reason[!tree$is_leaf]))) + }) + + test_that("tree, subnetworks, and edge membership are consistent", { + forbid_network() + corpus <- make_theme_corpus() + h <- decomposeSubnetworkIntoHierarchicalTopics( + corpus$subnetwork, max_edges = 10, n_topics = 3, + evidence = corpus$evidence, abstracts = corpus$abstracts + ) + tree <- h$tree + expect_setequal(names(h$subnetworks), tree$id) + expect_true(all(tree$parent_id[-1] %in% tree$id)) + expect_equal(unname(vapply(h$subnetworks[tree$id], + function(s) nrow(s$edges), integer(1))), + tree$n_edges) + + # Children are strict subsets of their parent's edges. + for (i in which(!is.na(tree$parent_id))) { + child <- h$subnetworks[[tree$id[i]]]$edges + parent <- h$subnetworks[[tree$parent_id[i]]]$edges + expect_true(all(child$stmt_hash %in% parent$stmt_hash)) + expect_lt(nrow(child), nrow(parent)) + } + + expect_equal(nrow(h$edge_membership), sum(tree$n_edges[-1])) + expect_equal(h$tree$pathString[1], "root") + child_row <- which(tree$depth == 1)[1] + expect_equal(tree$pathString[child_row], + paste0("root/", tree$id[child_row])) + }) + + test_that("queries INDRA and PubMed once when no corpus is supplied", { + corpus <- make_theme_corpus() + calls <- c(indra = 0, pubmed = 0) + testthat::local_mocked_bindings( + .extract_evidence_text = function(df) { + calls[["indra"]] <<- calls[["indra"]] + 1 + corpus$evidence + }, + .fetch_clean_abstracts_xml = function(pmids, ...) { + calls[["pubmed"]] <<- calls[["pubmed"]] + 1 + as.list(stats::setNames(corpus$abstracts[pmids], pmids)) + } + ) + h <- decomposeSubnetworkIntoHierarchicalTopics( + corpus$subnetwork, max_edges = 10, n_topics = 3 + ) + expect_gt(nrow(h$tree), 1) + expect_equal(calls, c(indra = 1, pubmed = 1)) + }) + + test_that("max_depth = 0 returns only the root", { + forbid_network() + corpus <- make_theme_corpus() + h <- decomposeSubnetworkIntoHierarchicalTopics( + corpus$subnetwork, max_depth = 0, + evidence = corpus$evidence, abstracts = corpus$abstracts + ) + expect_equal(nrow(h$tree), 1) + expect_equal(h$tree$stop_reason, "max_depth") + expect_equal(nrow(h$edge_membership), 0) + }) + + test_that("a small network is a single small_enough leaf", { + forbid_network() + corpus <- make_theme_corpus(edges_per_theme = 3) + h <- decomposeSubnetworkIntoHierarchicalTopics( + corpus$subnetwork, max_edges = 10, + evidence = corpus$evidence, abstracts = corpus$abstracts + ) + expect_equal(nrow(h$tree), 1) + expect_equal(h$tree$stop_reason, "small_enough") + }) + + test_that("print shows an indented tree", { + forbid_network() + corpus <- make_theme_corpus() + h <- decomposeSubnetworkIntoHierarchicalTopics( + corpus$subnetwork, max_edges = 10, n_topics = 3, + evidence = corpus$evidence, abstracts = corpus$abstracts + ) + out <- capture.output(print(h)) + expect_match(out[1], "^Topic hierarchy:") + expect_match(out[2], "^root \\[45 edges") + expect_true(any(grepl("^ 1 \\[", out))) + }) + + test_that("validates max_edges and max_depth", { + corpus <- make_theme_corpus() + expect_error( + decomposeSubnetworkIntoHierarchicalTopics(corpus$subnetwork, + max_edges = 0), + "`max_edges`" + ) + expect_error( + decomposeSubnetworkIntoHierarchicalTopics(corpus$subnetwork, + max_depth = 1.5), + "`max_depth`" + ) + }) +})