diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs index f9567a3ea4b..4d92b926c19 100644 --- a/rust/lance-index/src/scalar/inverted/builder.rs +++ b/rust/lance-index/src/scalar/inverted/builder.rs @@ -1254,7 +1254,7 @@ impl IndexWorker { .filter_map(|(doc, row_id)| doc.map(|doc| (doc, *row_id))); for (doc, row_id) in docs { - self.process_document(row_id, DocumentSource::Text(doc), false) + self.process_document(row_id, DocumentSource::Text(doc)) .await?; } } @@ -1298,7 +1298,7 @@ impl IndexWorker { continue; }; - self.process_document(*row_id, DocumentSource::StringList(doc.as_ref()), true) + self.process_document(*row_id, DocumentSource::StringList(doc.as_ref())) .await?; } @@ -1324,12 +1324,7 @@ impl IndexWorker { doc } - async fn process_document( - &mut self, - row_id: u64, - document: DocumentSource<'_>, - skip_empty_document: bool, - ) -> Result<()> { + async fn process_document(&mut self, row_id: u64, document: DocumentSource<'_>) -> Result<()> { let with_position = self.has_position(); let builder_was_empty = self.builder.docs.is_empty(); let old_temporary_memory_size = self.temporary_memory_size(); @@ -1435,7 +1430,11 @@ impl IndexWorker { self.builder.tokens.memory_size() as u64, ); - if skip_empty_document && token_num == 0 { + // A document that produces no tokens (e.g. an empty or whitespace-only + // string, or a list whose elements tokenize to nothing) is not indexed: + // it carries no searchable content, so counting it would only inflate + // `num_docs` and skew document-count-based statistics such as BM25. + if token_num == 0 { self.last_token_count = 0; self.trim_temporary_buffers(); self.adjust_tracked_memory_size( diff --git a/rust/lance/src/dataset/tests/dataset_index.rs b/rust/lance/src/dataset/tests/dataset_index.rs index 86682004f04..e9fadb5c33e 100644 --- a/rust/lance/src/dataset/tests/dataset_index.rs +++ b/rust/lance/src/dataset/tests/dataset_index.rs @@ -402,10 +402,18 @@ async fn test_create_fts_index_with_empty_strings() { false, )])); + // Mix documents that produce tokens with ones that tokenize to nothing + // (empty and whitespace-only strings). The latter carry no searchable + // content and must not be counted in `num_docs`. let batches: Vec = vec![ RecordBatch::try_new( schema.clone(), - vec![Arc::new(StringArray::from(vec!["", "", ""]))], + vec![Arc::new(StringArray::from(vec![ + "lance", + "", + "lance database", + " ", + ]))], ) .unwrap(), ]; @@ -420,6 +428,16 @@ async fn test_create_fts_index_with_empty_strings() { .await .unwrap(); + // Only the two token-producing rows are indexed; the empty and + // whitespace-only rows are skipped so they do not inflate BM25 statistics. + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics("text_idx").await.unwrap()).unwrap(); + assert_eq!( + stats["indices"][0]["num_docs"], 2, + "empty documents must not count: {stats}" + ); + + // The token-producing rows remain searchable. let batch = dataset .scan() .full_text_search(FullTextSearchQuery::new("lance".to_owned())) @@ -427,6 +445,16 @@ async fn test_create_fts_index_with_empty_strings() { .try_into_batch() .await .unwrap(); + assert_eq!(batch.num_rows(), 2); + + // A term present in no document returns nothing. + let batch = dataset + .scan() + .full_text_search(FullTextSearchQuery::new("missing".to_owned())) + .unwrap() + .try_into_batch() + .await + .unwrap(); assert_eq!(batch.num_rows(), 0); }