Skip to content

Add CorpusStorage::find_extra returning the query alternative for each match #342

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
matthias-stemmler wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add CorpusStorage::find_extra returning the query alternative for each match #342

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

- Added the `CorpusStorage::find_extra` method, which returns for each match the
alternative of the query that produced it in addition to the match ID.

## [4.1.5] - 2026-06-25

### Fixed
Expand Down
124 changes: 86 additions & 38 deletions graphannis/src/annis/db/corpusstorage.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use crate::annis::types::{
CorpusConfiguration, CorpusSizeUnit, FrequencyTable, FrequencyTableRow,
QueryAttributeDescription,
};
use crate::annis::types::{CorpusSizeInfo, CountExtra};
use crate::annis::types::{CorpusSizeInfo, CountExtra, MatchExtra};
use crate::annis::util::TimeoutCheck;
use crate::annis::util::quicksort;
use crate::{AnnotationGraph, graph::Match};
Expand Down Expand Up @@ -406,7 +406,9 @@ fn new_vector_with_memory_aligned_capacity<T>(expected_len: usize) -> Vec<T> {
Vec::with_capacity(aligned_memory_size / std::mem::size_of::<T>())
}

type FindIterator<'a> = Box<dyn Iterator<Item = Result<MatchGroup>> + 'a>;
/// Iterator over the matches of a `find` query
/// and the index of the query alternative that produced each match.
type FindIterator<'a> = Box<dyn Iterator<Item = Result<(usize, MatchGroup)>> + 'a>;

impl CorpusStorage {
/// Create a new instance with a maximum size for the internal corpus cache.
Expand Down Expand Up @@ -1735,36 +1737,36 @@ impl CorpusStorage {
// If the output is already sorted correctly, directly return the iterator.
// Quirks mode may change the order of the results, thus don't use the shortcut
// if quirks mode is active.
Box::from(plan)
Box::from(plan.matches_with_alternative())
} else {
let estimated_result_size = plan.estimated_output_size();
let btree_config = BtreeConfig::default()
.fixed_key_size(size_of::<usize>())
.max_value_size(512);
let mut anno_key_symbols: SymbolTable<AnnoKey> = SymbolTable::new();
let mut tmp_results: BtreeIndex<usize, Vec<(NodeID, usize)>> =
let mut tmp_results: BtreeIndex<usize, (usize, Vec<(NodeID, usize)>)> =
BtreeIndex::with_capacity(btree_config, estimated_result_size)?;

if find_arguments.order == ResultOrder::Randomized {
// Use a unique random index for each match to force a random order
let mut rng = rand::rng();

for mgroup in plan {
let mgroup = mgroup?;
for m in plan.matches_with_alternative() {
let (alternative, mgroup) = m?;
let mut idx: u64 = rng.random();
while tmp_results.contains_key(&(idx as usize))? {
idx = rng.random();
}
let m = match_group_with_symbol_ids(&mgroup, &mut anno_key_symbols)?;
tmp_results.insert(idx as usize, m)?;
tmp_results.insert(idx as usize, (alternative, m))?;
}
} else {
// Insert results in the order as they are given by the iterator
for (idx, mgroup) in plan.enumerate() {
let mgroup = mgroup?;
for (idx, m) in plan.matches_with_alternative().enumerate() {
let (alternative, mgroup) = m?;
// add all matches to temporary container
let m = match_group_with_symbol_ids(&mgroup, &mut anno_key_symbols)?;
tmp_results.insert(idx, m)?;
tmp_results.insert(idx, (alternative, m))?;
}

let token_helper = TokenHelper::new(db).ok();
Expand All @@ -1782,12 +1784,12 @@ impl CorpusStorage {

let gs_order = db.get_graphstorage(&component_order);
let mut cache = SortCache::new(gs_order);
let order_func = |m1: &Vec<(NodeID, usize)>,
m2: &Vec<(NodeID, usize)>|
let order_func = |m1: &(usize, Vec<(NodeID, usize)>),
m2: &(usize, Vec<(NodeID, usize)>)|
-> Result<std::cmp::Ordering> {
// Get matches from symbol ID
let m1 = match_group_resolve_symbol_ids(m1, &anno_key_symbols)?;
let m2 = match_group_resolve_symbol_ids(m2, &anno_key_symbols)?;
let m1 = match_group_resolve_symbol_ids(&m1.1, &anno_key_symbols)?;
let m2 = match_group_resolve_symbol_ids(&m2.1, &anno_key_symbols)?;

// Compare the matches
if find_arguments.order == ResultOrder::Inverted {
Expand Down Expand Up @@ -1826,31 +1828,28 @@ impl CorpusStorage {
quicksort::sort_first_n_items(&mut tmp_results, sort_size, order_func)?;
}
expected_size = Some(tmp_results.len());
let iterator = tmp_results.into_iter()?.map(move |unresolved_match_group| {
match unresolved_match_group {
Ok((_idx, unresolved_match_group)) => {
let result = match_group_resolve_symbol_ids(
&unresolved_match_group,
&anno_key_symbols,
)?;
Ok(result)
}
Err(e) => Err(e.into()),
let iterator = tmp_results.into_iter()?.map(move |entry| match entry {
Ok((_idx, (alternative, unresolved_match_group))) => {
let result =
match_group_resolve_symbol_ids(&unresolved_match_group, &anno_key_symbols)?;
Ok((alternative, result))
}
Err(e) => Err(e.into()),
});
Box::from(iterator)
};

Ok((base_it, expected_size))
}

fn find_in_single_corpus<S: AsRef<str>>(
fn find_in_single_corpus<S: AsRef<str>, T>(
&self,
query: &SearchQuery<S>,
corpus_name: &str,
find_arguments: FindArguments,
timeout: TimeoutCheck,
) -> Result<(Vec<String>, usize)> {
map_match: impl Fn(MatchExtra) -> T,
) -> Result<(Vec<T>, usize)> {
let prep = self.prepare_query(corpus_name, query.query, query.query_language, |db| {
let mut additional_components = vec![Component::new(
AnnotationComponentType::Ordering,
Expand Down Expand Up @@ -1884,7 +1883,7 @@ impl CorpusStorage {
timeout,
)?;

let mut results: Vec<String> = if let Some(expected_size) = expected_size {
let mut results: Vec<T> = if let Some(expected_size) = expected_size {
new_vector_with_memory_aligned_capacity(expected_size)
} else if let Some(limit) = find_arguments.limit {
new_vector_with_memory_aligned_capacity(limit)
Expand All @@ -1901,15 +1900,14 @@ impl CorpusStorage {
timeout.check()?;
}
}
let base_it: Box<dyn Iterator<Item = Result<MatchGroup>>> =
if let Some(limit) = find_arguments.limit {
Box::new(base_it.take(limit))
} else {
Box::new(base_it)
};
let base_it: FindIterator = if let Some(limit) = find_arguments.limit {
Box::new(base_it.take(limit))
} else {
Box::new(base_it)
};

for (match_nr, m) in base_it.enumerate() {
let m = m?;
let (alternative, m) = m?;
let mut match_desc = String::new();

let mut any_nodes_added = false;
Expand Down Expand Up @@ -1971,7 +1969,10 @@ impl CorpusStorage {
}
}
}
results.push(match_desc);
results.push(map_match(MatchExtra {
match_id: match_desc,
alternative,
}));
if match_nr % 1_000 == 0 {
timeout.check()?;
}
Expand All @@ -1991,13 +1992,49 @@ impl CorpusStorage {
///
/// Returns a vector of match IDs, where each match ID consists of the matched node annotation identifiers separated by spaces.
/// You can use the [subgraph(...)](#method.subgraph) method to get the subgraph for a single match described by the node annnotation identifiers.
///
/// In order to obtain additional information for each match, see the [find_extra(...)](#method.find_extra) method.
pub fn find<S: AsRef<str>>(
&self,
query: SearchQuery<S>,
offset: usize,
limit: Option<usize>,
order: ResultOrder,
) -> Result<Vec<String>> {
self.find_mapped(query, offset, limit, order, |m| m.match_id)
}

/// Find all results for a `query` and return the match ID and additional information for each result.
///
/// The query is paginated and an offset and limit can be specified.
///
/// - `query` - The search query definition.
/// - `offset` - Skip the `n` first results, where `n` is the offset.
/// - `limit` - Return at most `n` matches, where `n` is the limit. Use `None` to allow unlimited result sizes.
/// - `order` - Specify the order of the matches.
///
/// Returns a vector of [`MatchExtra`], which contains the same match ID as returned by [find(...)](#method.find)
/// together with additional information about the match.
///
/// If you are just interested in the match IDs, use the [find(...)](#method.find) method instead.
pub fn find_extra<S: AsRef<str>>(
&self,
query: SearchQuery<S>,
offset: usize,
limit: Option<usize>,
order: ResultOrder,
) -> Result<Vec<MatchExtra>> {
self.find_mapped(query, offset, limit, order, |m| m)
}

fn find_mapped<S: AsRef<str>, T: Clone>(
&self,
query: SearchQuery<S>,
offset: usize,
limit: Option<usize>,
order: ResultOrder,
map_match: impl Fn(MatchExtra) -> T,
) -> Result<Vec<T>> {
let timeout = TimeoutCheck::new(query.timeout);

// Sort corpus names
Expand All @@ -2016,7 +2053,13 @@ impl CorpusStorage {
match corpus_names.len() {
0 => Ok(Vec::new()),
1 => self
.find_in_single_corpus(&query, corpus_names[0].as_str(), find_arguments, timeout)
.find_in_single_corpus(
&query,
corpus_names[0].as_str(),
find_arguments,
timeout,
map_match,
)
.map(|r| r.0),
_ => {
if order == ResultOrder::Randomized {
Expand All @@ -2033,8 +2076,13 @@ impl CorpusStorage {

let mut result = Vec::new();
for cn in corpus_names {
let (single_result, skipped) =
self.find_in_single_corpus(&query, cn.as_ref(), find_arguments, timeout)?;
let (single_result, skipped) = self.find_in_single_corpus(
&query,
cn.as_ref(),
find_arguments,
timeout,
&map_match,
)?;

// Adjust limit and offset according to the found matches for the next corpus.
let single_result_length = single_result.len();
Expand Down
58 changes: 58 additions & 0 deletions .../snapshots/graphannis__annis__db__corpusstorage__tests__find_extra_with_alternatives.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
---
source: graphannis/src/annis/db/corpusstorage/tests.rs
expression: results
---
[
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus1/doc1#sTok1",
alternative: 0,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus1/doc1#sTok3",
alternative: 1,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus1/doc1#sTok5",
alternative: 1,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus1/doc1#sTok8",
alternative: 0,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus1/doc2#sTok1",
alternative: 0,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus1/doc2#sTok3",
alternative: 1,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus1/doc2#sTok8",
alternative: 0,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus2/doc3#sTok1",
alternative: 0,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus2/doc3#sTok3",
alternative: 1,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus2/doc3#sTok8",
alternative: 0,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus2/doc4#sTok1",
alternative: 0,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus2/doc4#sTok3",
alternative: 1,
},
MatchExtra {
match_id: "salt::pos::rootCorpus/subCorpus2/doc4#sTok8",
alternative: 0,
},
]
Loading
Loading