From f06d8a8e37a6d4c3885c27ce14271e31e74cc44a Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Mon, 2 Nov 2026 21:10:59 +0100 Subject: [PATCH 1/7] feat: match word prefixes in full-text search --- CHANGELOG.md | 2 + src/internal/search/mod.rs | 160 ++++++++++++++++++++++++++++++++----- 2 files changed, 144 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03c08ea..df8c506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ All notable changes to Craqle are documented here. `WalAlreadyDurable` sets the policy locally without publishing it or creating the graph topic. - `CraqleNode::applied_cursor_digest` returns the digest `repair_irokle_topic_cursor` expects, so a fenced topic can be repaired on a running node. +- Full-text search also matches indexed words that start with a query word of at least three + characters, so `SAMD` finds `SAMD00012345`. Exact word matches rank above such prefix matches. ### Changed diff --git a/src/internal/search/mod.rs b/src/internal/search/mod.rs index f96a406..7b1c3e9 100644 --- a/src/internal/search/mod.rs +++ b/src/internal/search/mod.rs @@ -8,6 +8,7 @@ use std::borrow::Cow; use std::collections::BinaryHeap; use std::collections::{BTreeMap, HashMap, HashSet}; use std::io::Write; +use std::ops::Bound; use std::path::Path; #[cfg(test)] use std::sync::Condvar; @@ -18,15 +19,15 @@ use tantivy::TERMINATED; #[cfg(test)] use tantivy::collector::{BytesFilterCollector, TopDocs}; use tantivy::query::{ - AllQuery, Bm25StatisticsProvider as _, Bm25Weight, BooleanQuery, EnableScoring, Occur, Query, - QueryParser, TermQuery, TermSetQuery, + AllQuery, Bm25StatisticsProvider as _, Bm25Weight, BooleanQuery, BoostQuery, EnableScoring, + Occur, Query, QueryParser, RangeQuery, TermQuery, TermSetQuery, }; use tantivy::schema::{ FAST, Field, IndexRecordOption, STORED, STRING, Schema, SchemaBuilder, TEXT, TextFieldIndexing, Value, }; use tantivy::tokenizer::{ - AsciiFoldingFilter, LowerCaser, RemoveLongFilter, SimpleTokenizer, TextAnalyzer, + AsciiFoldingFilter, LowerCaser, RemoveLongFilter, SimpleTokenizer, TextAnalyzer, TokenStream, }; use tantivy::{DocAddress, DocId, DocSet, Index, IndexReader, IndexWriter, Score, Searcher}; use tantivy::{TantivyDocument, Term}; @@ -55,6 +56,10 @@ const REBUILD_SHARDS: usize = 64; /// Damaged graphs retained for targeted reindex; overflow escalates to a whole rebuild. const DAMAGED_GRAPHS: usize = 64; const ALL_TEXT_TOKENIZER: &str = "craqle_text_v2"; +/// Query words this long also match the indexed words that start with them. +const PREFIX_CHARS: usize = 3; +/// Prefix matches add this constant score, so an exact match of a word ranks first. +const PREFIX_BOOST: Score = 0.01; const INDEX_VERSION_FIELD: &str = "_craqle_search_index_v5"; const INDEX_ID_FILE: &str = ".craqle-index-id"; const DIRECT_GENERATION: GenerationId = GenerationId(1); @@ -1866,10 +1871,20 @@ impl SearchIndex { let view = self.pin_view(); self.check_bound(&view).map_err(E::from)?; let parser = QueryParser::for_index(&self.index, vec![self.f_all_text]); - let parsed = parser - .parse_query(&sanitize_query(req.query)) + let sanitized = sanitize_query(req.query); + let mut parsed = parser + .parse_query(&sanitized) .map_err(SearchError::from) .map_err(E::from)?; + // Complete matching serves SPARQL text filters, which keep whole-word semantics. + let prefixes = match complete { + true => Vec::new(), + false => (self.prefix_clauses(&view.searcher, &sanitized)).map_err(E::from)?, + }; + if !prefixes.is_empty() { + let clauses = std::iter::once((Occur::Should, parsed)).chain(prefixes); + parsed = Box::new(BooleanQuery::new(clauses.collect())); + } let mut query: Box = if let Some(subject) = req.subject { Box::new(BooleanQuery::new(vec![ (Occur::Must, parsed), @@ -1896,19 +1911,7 @@ impl SearchIndex { && let Some(subjects) = prepare()? { (req.check)()?; - let bitset = view - .searcher - .segment_readers() - .iter() - .map(|reader| { - (reader.max_doc() as usize) - .div_ceil(64) - .saturating_mul(8) - .saturating_add(64) - }) - .max() - .unwrap_or(0); - let bytes = candidate_bytes(&subjects).saturating_add(bitset); + let bytes = candidate_bytes(&subjects).saturating_add(bitset_bytes(&view.searcher)); let limit = req.limit.min(crate::MAX_SEARCH_LIMIT); // Postings frequency is a whole-index count that could span graphs this caller // cannot see, so admission relies only on the already-authorized candidate set. @@ -3307,6 +3310,39 @@ impl SearchIndex { }) } + /// Matches on the longer indexed words that start with each long enough query word; none + /// when their per-segment bitsets would exceed a quarter of the query budget. + fn prefix_clauses( + &self, + searcher: &Searcher, + query: &str, + ) -> Result)>> { + let mut analyzer = self.index.tokenizer_for_field(self.f_all_text)?; + let mut stream = analyzer.token_stream(query); + let mut words = std::collections::BTreeSet::new(); + while let Some(token) = stream.next() { + if token.text.chars().count() >= PREFIX_CHARS + && has_longer(searcher, self.f_all_text, &token.text)? + { + words.insert(token.text.clone()); + } + } + if words.len().saturating_mul(bitset_bytes(searcher)) > self.query_bytes() / 4 { + return Ok(Vec::new()); + } + let field = self.f_all_text; + let clauses = words.into_iter().map(|word| { + let upper = prefix_end(word.as_bytes()).map_or(Bound::Unbounded, |end| { + Bound::Excluded(Term::from_field_bytes(field, &end)) + }); + let start = Bound::Included(Term::from_field_text(field, &word)); + let range: Box = Box::new(RangeQuery::new(start, upper)); + let boosted: Box = Box::new(BoostQuery::new(range, PREFIX_BOOST)); + (Occur::Should, boosted) + }); + Ok(clauses.collect()) + } + /// Whether block-max pruning bounds hold: one segment, one field, and a term union /// scored with that segment's own statistics. fn pruning_safe(&self, searcher: &Searcher, query: &dyn Query) -> bool { @@ -3821,6 +3857,45 @@ fn rebuild_shard(graph_iri: &str) -> usize { (hasher.finish() % REBUILD_SHARDS as u64) as usize } +/// Bytes of the largest segment's document bitset. +fn bitset_bytes(searcher: &Searcher) -> usize { + searcher + .segment_readers() + .iter() + .map(|reader| { + (reader.max_doc() as usize) + .div_ceil(64) + .saturating_mul(8) + .saturating_add(64) + }) + .max() + .unwrap_or(0) +} + +/// Whether an indexed word of `field` starts with `word` and is longer than it. +fn has_longer(searcher: &Searcher, field: Field, word: &str) -> Result { + let end = prefix_end(word.as_bytes()); + for reader in searcher.segment_readers() { + let inverted = reader.inverted_index(field)?; + let mut range = inverted.terms().range().gt(word.as_bytes()); + if let Some(end) = &end { + range = range.lt(end); + } + if range.into_stream()?.advance() { + return Ok(true); + } + } + Ok(false) +} + +/// The smallest byte string after every string that starts with `prefix`. +fn prefix_end(prefix: &[u8]) -> Option> { + let last = prefix.iter().rposition(|byte| *byte != u8::MAX)?; + let mut end = prefix[..=last].to_vec(); + end[last] += 1; + Some(end) +} + fn sanitize_query(query: &str) -> String { let cleaned: String = query .chars() @@ -4175,6 +4250,55 @@ mod tests { assert!(bounded.rows < page.rows); } + fn prefix_fixture(descriptions: &[(&str, &str)]) -> (tempfile::TempDir, crate::CraqleNode) { + let dir = tempdir().unwrap(); + let node = crate::CraqleNode::open(dir.path()).unwrap(); + for (graph, description) in descriptions { + let request = crate_request(&GraphId::new(graph), description, true); + node.create_crate(&writer_auth(), request).unwrap(); + } + node.flush_search_updates().unwrap(); + (dir, node) + } + + #[test] + fn prefix_finds_accessions() { + let (_dir, node) = prefix_fixture(&[("urn:test:accession", "sample SAMD00012345")]); + let hits = node.search.search("SAMD", 10).unwrap(); + assert!(!hits.is_empty()); + assert!(hits.iter().all(|hit| hit.graph_id == "urn:test:accession")); + assert!(node.search.search("sa", 10).unwrap().is_empty()); + } + + #[test] + fn exact_ranks_first() { + let (_dir, node) = prefix_fixture(&[ + ("urn:test:longer", "samd00099999"), + ("urn:test:exact", "samd"), + ]); + let hits = node.search.search("samd", 10).unwrap(); + let position = |graph: &str| hits.iter().position(|hit| hit.graph_id == graph); + let last_exact = hits + .iter() + .rposition(|hit| hit.graph_id == "urn:test:exact"); + assert!(position("urn:test:longer").is_some()); + assert!(last_exact.unwrap() < position("urn:test:longer").unwrap()); + } + + #[test] + fn prefix_ignores_patterns() { + let (_dir, node) = prefix_fixture(&[("urn:test:pattern", "SAMD00012345")]); + for query in [ + ".*00012345", + "s[a-z]md00012345", + "(sa|xx)md", + "s.m.00012345", + ] { + assert!(node.search.search(query, 10).unwrap().is_empty(), "{query}"); + } + assert!(!node.search.search("samd0001", 10).unwrap().is_empty()); + } + #[test] fn search_keeps_identifiers() { let dir = tempdir().unwrap(); From 83f88a650b959ef8e6c0d6d0fc3d6b5c3756d31c Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Mon, 2 Nov 2026 22:43:44 +0100 Subject: [PATCH 2/7] fix: rank prefix-only search hits after exact matches --- src/internal/search/mod.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/internal/search/mod.rs b/src/internal/search/mod.rs index 7b1c3e9..38414e1 100644 --- a/src/internal/search/mod.rs +++ b/src/internal/search/mod.rs @@ -58,8 +58,9 @@ const DAMAGED_GRAPHS: usize = 64; const ALL_TEXT_TOKENIZER: &str = "craqle_text_v2"; /// Query words this long also match the indexed words that start with them. const PREFIX_CHARS: usize = 3; -/// Prefix matches add this constant score, so an exact match of a word ranks first. -const PREFIX_BOOST: Score = 0.01; +/// Prefix matches add no score, so they rank after every exact match, whose BM25 score is +/// positive, and leave the order of exact matches unchanged. +const PREFIX_BOOST: Score = 0.0; const INDEX_VERSION_FIELD: &str = "_craqle_search_index_v5"; const INDEX_ID_FILE: &str = ".craqle-index-id"; const DIRECT_GENERATION: GenerationId = GenerationId(1); @@ -4285,6 +4286,27 @@ mod tests { assert!(last_exact.unwrap() < position("urn:test:longer").unwrap()); } + // Exact matches of one word rank above documents that match every word only by prefix. + #[test] + fn multiword_exact_first() { + let budget = MemoryBudget::new(1 << 30, 256 << 20).with_search_writer(20_000_000); + let index = SearchIndex::memory_with_budget(budget).unwrap(); + for position in 0..300 { + let subject = format!("urn:test:tier:{position}"); + index + .index_resource("urn:test:tier", &subject, Some("sample")) + .unwrap(); + } + let text = Some("SAMD00012345 samples"); + index + .index_resource("urn:test:tier", "urn:test:tier:prefix", text) + .unwrap(); + index.commit().unwrap(); + let hits = index.search("sample SAMD", 400).unwrap(); + assert_eq!(hits.len(), 301); + assert_eq!(hits[300].subject_iri, "urn:test:tier:prefix"); + } + #[test] fn prefix_ignores_patterns() { let (_dir, node) = prefix_fixture(&[("urn:test:pattern", "SAMD00012345")]); From 3782e9bc343315afd09222a01609088ddd6b2f8d Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Mon, 2 Nov 2026 22:48:58 +0100 Subject: [PATCH 3/7] fix: expand search prefixes in checked and budgeted steps --- src/internal/search/mod.rs | 205 +++++++++++++++++++++++++++++++------ 1 file changed, 176 insertions(+), 29 deletions(-) diff --git a/src/internal/search/mod.rs b/src/internal/search/mod.rs index 38414e1..b61ebf4 100644 --- a/src/internal/search/mod.rs +++ b/src/internal/search/mod.rs @@ -8,19 +8,20 @@ use std::borrow::Cow; use std::collections::BinaryHeap; use std::collections::{BTreeMap, HashMap, HashSet}; use std::io::Write; -use std::ops::Bound; use std::path::Path; #[cfg(test)] use std::sync::Condvar; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, LazyLock, Mutex, MutexGuard, PoisonError, RwLock}; +use tantivy::SegmentReader; use tantivy::TERMINATED; #[cfg(test)] use tantivy::collector::{BytesFilterCollector, TopDocs}; +use tantivy::index::SegmentId; use tantivy::query::{ AllQuery, Bm25StatisticsProvider as _, Bm25Weight, BooleanQuery, BoostQuery, EnableScoring, - Occur, Query, QueryParser, RangeQuery, TermQuery, TermSetQuery, + Explanation, Occur, Query, QueryParser, Scorer, TermQuery, TermSetQuery, Weight, }; use tantivy::schema::{ FAST, Field, IndexRecordOption, STORED, STRING, Schema, SchemaBuilder, TEXT, TextFieldIndexing, @@ -297,6 +298,82 @@ pub(crate) struct FilterQuery<'a, E> { pub candidates: Option<&'a CandidateSource<'a, E>>, } +/// A ranked search's query words to expand by prefix, with its cancellation check. +struct PrefixScan<'a, E> { + searcher: &'a Searcher, + query: &'a str, + check: &'a dyn Fn() -> std::result::Result<(), E>, +} + +/// The documents of each segment that match a query word by prefix, at constant score. +#[derive(Clone, Debug)] +struct PrefixDocs(Arc>>); + +impl Query for PrefixDocs { + fn weight(&self, _: EnableScoring<'_>) -> tantivy::Result> { + Ok(Box::new(self.clone())) + } +} + +impl Weight for PrefixDocs { + fn scorer(&self, reader: &SegmentReader, boost: Score) -> tantivy::Result> { + let docs = self + .0 + .get(&reader.segment_id()) + .cloned() + .unwrap_or_default(); + Ok(Box::new(SortedDocs { + docs, + cursor: 0, + score: boost, + })) + } + + fn explain(&self, reader: &SegmentReader, doc: DocId) -> tantivy::Result { + let docs = self.0.get(&reader.segment_id()); + match docs.is_some_and(|docs| docs.binary_search(&doc).is_ok()) { + true => Ok(Explanation::new("PrefixDocs", 1.0)), + false => Err(tantivy::TantivyError::InvalidArgument(format!( + "document {doc} does not match a prefix" + ))), + } + } +} + +/// One segment's prefix matches in document order, each with the same score. +struct SortedDocs { + docs: Vec, + cursor: usize, + score: Score, +} + +impl DocSet for SortedDocs { + fn advance(&mut self) -> DocId { + self.cursor = (self.cursor + 1).min(self.docs.len()); + self.doc() + } + + fn seek(&mut self, target: DocId) -> DocId { + let rest = self.docs.get(self.cursor..).unwrap_or_default(); + self.cursor += rest.partition_point(|doc| *doc < target); + self.doc() + } + + fn doc(&self) -> DocId { + self.docs.get(self.cursor).copied().unwrap_or(TERMINATED) + } + + fn size_hint(&self) -> u32 { + u32::try_from(self.docs.len()).unwrap_or(u32::MAX) + } +} + +impl Scorer for SortedDocs { + fn score(&mut self) -> Score { + self.score + } +} + pub(crate) type CandidateSource<'a, E> = dyn Fn() -> std::result::Result>, E> + 'a; @@ -1878,13 +1955,21 @@ impl SearchIndex { .map_err(SearchError::from) .map_err(E::from)?; // Complete matching serves SPARQL text filters, which keep whole-word semantics. + let scan = PrefixScan { + searcher: &view.searcher, + query: &sanitized, + check: req.check, + }; let prefixes = match complete { - true => Vec::new(), - false => (self.prefix_clauses(&view.searcher, &sanitized)).map_err(E::from)?, + true => None, + false => self.prefix_docs(scan)?, }; - if !prefixes.is_empty() { - let clauses = std::iter::once((Occur::Should, parsed)).chain(prefixes); - parsed = Box::new(BooleanQuery::new(clauses.collect())); + if let Some(prefixes) = prefixes { + let boosted: Box = Box::new(BoostQuery::new(prefixes, PREFIX_BOOST)); + parsed = Box::new(BooleanQuery::new(vec![ + (Occur::Should, parsed), + (Occur::Should, boosted), + ])); } let mut query: Box = if let Some(subject) = req.subject { Box::new(BooleanQuery::new(vec![ @@ -3311,37 +3396,64 @@ impl SearchIndex { }) } - /// Matches on the longer indexed words that start with each long enough query word; none - /// when their per-segment bitsets would exceed a quarter of the query budget. - fn prefix_clauses( + /// Documents holding a longer indexed word that starts with a long enough query word; none + /// without such a word or when the documents outgrow a quarter of the query budget. + fn prefix_docs( &self, - searcher: &Searcher, - query: &str, - ) -> Result)>> { - let mut analyzer = self.index.tokenizer_for_field(self.f_all_text)?; - let mut stream = analyzer.token_stream(query); + req: PrefixScan<'_, E>, + ) -> std::result::Result>, E> + where + E: From, + { + let mut analyzer = + (self.index.tokenizer_for_field(self.f_all_text)).map_err(SearchError::from)?; + let mut stream = analyzer.token_stream(req.query); let mut words = std::collections::BTreeSet::new(); while let Some(token) = stream.next() { if token.text.chars().count() >= PREFIX_CHARS - && has_longer(searcher, self.f_all_text, &token.text)? + && has_longer(req.searcher, self.f_all_text, &token.text)? { words.insert(token.text.clone()); } } - if words.len().saturating_mul(bitset_bytes(searcher)) > self.query_bytes() / 4 { - return Ok(Vec::new()); + if words.is_empty() { + return Ok(None); } - let field = self.f_all_text; - let clauses = words.into_iter().map(|word| { - let upper = prefix_end(word.as_bytes()).map_or(Bound::Unbounded, |end| { - Bound::Excluded(Term::from_field_bytes(field, &end)) - }); - let start = Bound::Included(Term::from_field_text(field, &word)); - let range: Box = Box::new(RangeQuery::new(start, upper)); - let boosted: Box = Box::new(BoostQuery::new(range, PREFIX_BOOST)); - (Occur::Should, boosted) - }); - Ok(clauses.collect()) + let (limit, mut bytes) = (self.query_bytes() / 4, 0usize); + let mut segments = HashMap::new(); + for reader in req.searcher.segment_readers() { + let inverted = (reader.inverted_index(self.f_all_text)).map_err(SearchError::from)?; + let mut docs = Vec::new(); + for word in &words { + let mut terms = inverted.terms().range().ge(word.as_bytes()); + if let Some(end) = prefix_end(word.as_bytes()) { + terms = terms.lt(end); + } + let mut terms = terms.into_stream().map_err(SearchError::from)?; + while terms.advance() { + (req.check)()?; + let option = IndexRecordOption::Basic; + let mut postings = (inverted + .read_block_postings_from_terminfo(terms.value(), option)) + .map_err(SearchError::from)?; + while !postings.docs().is_empty() { + (req.check)()?; + // A scorer copies one segment's list, so each document counts twice. + let block = std::mem::size_of_val(postings.docs()).saturating_mul(2); + bytes = bytes.saturating_add(block); + if bytes > limit { + return Ok(None); + } + docs.extend_from_slice(postings.docs()); + postings.advance(); + } + } + } + docs.sort_unstable(); + docs.dedup(); + segments.insert(reader.segment_id(), docs); + } + Ok(Some(Box::new(PrefixDocs(Arc::new(segments))))) } /// Whether block-max pruning bounds hold: one segment, one field, and a term union @@ -4307,6 +4419,41 @@ mod tests { assert_eq!(hits[300].subject_iri, "urn:test:tier:prefix"); } + // The prefix expansion checks for cancellation per word and postings block, so it stops there. + #[test] + fn prefix_expansion_cancels() { + let budget = MemoryBudget::new(1 << 30, 256 << 20).with_search_writer(20_000_000); + let index = SearchIndex::memory_with_budget(budget).unwrap(); + for position in 0..50 { + let text = format!("samd{position:08}"); + let subject = format!("urn:test:cancel:{position}"); + index + .index_resource("urn:test:cancel", &subject, Some(&text)) + .unwrap(); + } + index.commit().unwrap(); + let view = index.pin_view(); + let calls = AtomicU64::new(0); + let check = || match calls.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(()), + _ => Err(SearchError::Cancelled), + }; + let scan = PrefixScan { + searcher: &view.searcher, + query: "samd", + check: &check, + }; + assert!(matches!( + index.prefix_docs(scan), + Err(SearchError::Cancelled) + )); + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "the expansion stops at the failed check" + ); + } + #[test] fn prefix_ignores_patterns() { let (_dir, node) = prefix_fixture(&[("urn:test:pattern", "SAMD00012345")]); From 95125909b1cee4f61d652fbbf4bc92d45a57e288 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Mon, 2 Nov 2026 23:06:50 +0100 Subject: [PATCH 4/7] fix: fail prefix expansions over the query budget --- src/internal/search/mod.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/internal/search/mod.rs b/src/internal/search/mod.rs index b61ebf4..3ae808a 100644 --- a/src/internal/search/mod.rs +++ b/src/internal/search/mod.rs @@ -3397,7 +3397,7 @@ impl SearchIndex { } /// Documents holding a longer indexed word that starts with a long enough query word; none - /// without such a word or when the documents outgrow a quarter of the query budget. + /// without such a word. More documents than a quarter of the query budget are an error. fn prefix_docs( &self, req: PrefixScan<'_, E>, @@ -3442,7 +3442,7 @@ impl SearchIndex { let block = std::mem::size_of_val(postings.docs()).saturating_mul(2); bytes = bytes.saturating_add(block); if bytes > limit { - return Ok(None); + return Err(E::from(SearchError::ItemTooLarge { bytes, limit })); } docs.extend_from_slice(postings.docs()); postings.advance(); @@ -4419,6 +4419,28 @@ mod tests { assert_eq!(hits[300].subject_iri, "urn:test:tier:prefix"); } + // An expansion over its budget fails like other oversized results, never as exact-only hits. + #[test] + fn prefix_budget_fails() { + let budget = MemoryBudget::new(1 << 30, 256 << 20).with_search_writer(20_000_000); + let mut index = SearchIndex::memory_with_budget(budget).unwrap(); + for position in 0..200 { + let text = format!("samd{position:08}"); + let subject = format!("urn:test:budget:{position}"); + index + .index_resource("urn:test:budget", &subject, Some(&text)) + .unwrap(); + } + index.commit().unwrap(); + index.prepared_bytes = 16_384; + let result = index.search("samd", 10); + assert!( + matches!(result, Err(SearchError::ItemTooLarge { .. })), + "{result:?}" + ); + assert_eq!(index.search("samd00000001", 10).unwrap().len(), 1); + } + // The prefix expansion checks for cancellation per word and postings block, so it stops there. #[test] fn prefix_expansion_cancels() { From 06ef1bebeae787656dc814dad3d83ce1491ba53e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Mon, 2 Nov 2026 23:09:27 +0100 Subject: [PATCH 5/7] fix: charge exact prefix list capacity and share lists with scorers --- src/internal/search/mod.rs | 69 ++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/src/internal/search/mod.rs b/src/internal/search/mod.rs index 3ae808a..9fe9800 100644 --- a/src/internal/search/mod.rs +++ b/src/internal/search/mod.rs @@ -307,7 +307,7 @@ struct PrefixScan<'a, E> { /// The documents of each segment that match a query word by prefix, at constant score. #[derive(Clone, Debug)] -struct PrefixDocs(Arc>>); +struct PrefixDocs(Arc>>>); impl Query for PrefixDocs { fn weight(&self, _: EnableScoring<'_>) -> tantivy::Result> { @@ -342,7 +342,7 @@ impl Weight for PrefixDocs { /// One segment's prefix matches in document order, each with the same score. struct SortedDocs { - docs: Vec, + docs: Arc>, cursor: usize, score: Score, } @@ -3423,13 +3423,29 @@ impl SearchIndex { let mut segments = HashMap::new(); for reader in req.searcher.segment_readers() { let inverted = (reader.inverted_index(self.f_all_text)).map_err(SearchError::from)?; - let mut docs = Vec::new(); - for word in &words { - let mut terms = inverted.terms().range().ge(word.as_bytes()); + let terms = |word: &str| { + let mut range = inverted.terms().range().ge(word.as_bytes()); if let Some(end) = prefix_end(word.as_bytes()) { - terms = terms.lt(end); + range = range.lt(end); + } + range.into_stream().map_err(SearchError::from) + }; + // The postings' document frequencies size the list once, so it never grows. + let mut size = 0usize; + for word in &words { + let mut terms = terms(word)?; + while terms.advance() { + (req.check)()?; + size = size.saturating_add(terms.value().doc_freq as usize); } - let mut terms = terms.into_stream().map_err(SearchError::from)?; + } + bytes = bytes.saturating_add(size.saturating_mul(std::mem::size_of::())); + if bytes > limit { + return Err(E::from(SearchError::ItemTooLarge { bytes, limit })); + } + let mut docs = Vec::with_capacity(size); + for word in &words { + let mut terms = terms(word)?; while terms.advance() { (req.check)()?; let option = IndexRecordOption::Basic; @@ -3438,11 +3454,10 @@ impl SearchIndex { .map_err(SearchError::from)?; while !postings.docs().is_empty() { (req.check)()?; - // A scorer copies one segment's list, so each document counts twice. - let block = std::mem::size_of_val(postings.docs()).saturating_mul(2); - bytes = bytes.saturating_add(block); - if bytes > limit { - return Err(E::from(SearchError::ItemTooLarge { bytes, limit })); + if docs.len().saturating_add(postings.docs().len()) > size { + return Err(E::from(SearchError::Damaged { + detail: "postings length", + })); } docs.extend_from_slice(postings.docs()); postings.advance(); @@ -3451,7 +3466,7 @@ impl SearchIndex { } docs.sort_unstable(); docs.dedup(); - segments.insert(reader.segment_id(), docs); + segments.insert(reader.segment_id(), Arc::new(docs)); } Ok(Some(Box::new(PrefixDocs(Arc::new(segments))))) } @@ -4432,7 +4447,7 @@ mod tests { .unwrap(); } index.commit().unwrap(); - index.prepared_bytes = 16_384; + index.prepared_bytes = 12_796; let result = index.search("samd", 10); assert!( matches!(result, Err(SearchError::ItemTooLarge { .. })), @@ -4441,6 +4456,32 @@ mod tests { assert_eq!(index.search("samd00000001", 10).unwrap().len(), 1); } + // The expansion charges its lists' exact capacity: 200 matches cost 800 bytes, no more. + #[test] + fn prefix_budget_exact() { + let budget = MemoryBudget::new(1 << 30, 256 << 20).with_search_writer(20_000_000); + let mut index = SearchIndex::memory_with_budget(budget).unwrap(); + for position in 0..200 { + let text = format!("samd{position:08}"); + let subject = format!("urn:test:exact-budget:{position}"); + index + .index_resource("urn:test:exact-budget", &subject, Some(&text)) + .unwrap(); + } + index.commit().unwrap(); + index.prepared_bytes = 12_800; + assert_eq!(index.search("samd", 10).unwrap().len(), 10); + index.prepared_bytes = 12_796; + let result = index.search("samd", 10); + assert!(matches!( + result, + Err(SearchError::ItemTooLarge { + bytes: 800, + limit: 799 + }) + )); + } + // The prefix expansion checks for cancellation per word and postings block, so it stops there. #[test] fn prefix_expansion_cancels() { From a68de26c1caf7c646bd9cb07e2980845968c01e3 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Mon, 2 Nov 2026 23:11:47 +0100 Subject: [PATCH 6/7] fix: keep exact search scores above prefix-only scores --- src/internal/search/mod.rs | 44 +++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/src/internal/search/mod.rs b/src/internal/search/mod.rs index 9fe9800..5703e85 100644 --- a/src/internal/search/mod.rs +++ b/src/internal/search/mod.rs @@ -20,8 +20,8 @@ use tantivy::TERMINATED; use tantivy::collector::{BytesFilterCollector, TopDocs}; use tantivy::index::SegmentId; use tantivy::query::{ - AllQuery, Bm25StatisticsProvider as _, Bm25Weight, BooleanQuery, BoostQuery, EnableScoring, - Explanation, Occur, Query, QueryParser, Scorer, TermQuery, TermSetQuery, Weight, + AllQuery, Bm25StatisticsProvider as _, Bm25Weight, BooleanQuery, BoostQuery, ConstScoreQuery, + EnableScoring, Explanation, Occur, Query, QueryParser, Scorer, TermQuery, TermSetQuery, Weight, }; use tantivy::schema::{ FAST, Field, IndexRecordOption, STORED, STRING, Schema, SchemaBuilder, TEXT, TextFieldIndexing, @@ -1967,7 +1967,7 @@ impl SearchIndex { if let Some(prefixes) = prefixes { let boosted: Box = Box::new(BoostQuery::new(prefixes, PREFIX_BOOST)); parsed = Box::new(BooleanQuery::new(vec![ - (Occur::Should, parsed), + (Occur::Should, floor_exact(parsed)), (Occur::Should, boosted), ])); } @@ -3985,6 +3985,16 @@ fn rebuild_shard(graph_iri: &str) -> usize { (hasher.finish() % REBUILD_SHARDS as u64) as usize } +/// Raises every exact match's score above zero, the score of prefix-only matches. The floor is +/// lost in any normal score, so it only lifts scores that round to zero. +fn floor_exact(exact: Box) -> Box { + let floor = Box::new(ConstScoreQuery::new(exact.box_clone(), Score::MIN_POSITIVE)); + Box::new(BooleanQuery::new(vec![ + (Occur::Should, exact), + (Occur::Should, floor), + ])) +} + /// Bytes of the largest segment's document bitset. fn bitset_bytes(searcher: &Searcher) -> usize { searcher @@ -4482,6 +4492,34 @@ mod tests { )); } + // An exact match whose BM25 score rounds to zero still ranks above a prefix-only match. + #[test] + fn exact_floor_ranks() { + let budget = MemoryBudget::new(1 << 30, 256 << 20).with_search_writer(20_000_000); + let index = SearchIndex::memory_with_budget(budget).unwrap(); + index + .index_resource("urn:test:floor", "urn:test:floor:exact", Some("samd")) + .unwrap(); + let text = Some("samd00012345"); + index + .index_resource("urn:test:floor", "urn:test:floor:prefix", text) + .unwrap(); + index.commit().unwrap(); + let term = |text: &str| -> Box { + let term = Term::from_field_text(index.f_all_text, text); + let term = Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs)); + Box::new(BoostQuery::new(term, 0.0)) + }; + let query = BooleanQuery::new(vec![ + (Occur::Should, floor_exact(term("samd"))), + (Occur::Should, term("samd00012345")), + ]); + let searcher = &index.pin_view().searcher; + let top = searcher.search(&query, &TopDocs::with_limit(2).order_by_score()); + let scores: Vec = top.unwrap().into_iter().map(|(score, _)| score).collect(); + assert!(scores[0] > 0.0 && scores[1] == 0.0, "{scores:?}"); + } + // The prefix expansion checks for cancellation per word and postings block, so it stops there. #[test] fn prefix_expansion_cancels() { From c869a36e6e54fafa502491140c5f9ed8f195b018 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Mon, 2 Nov 2026 23:29:34 +0100 Subject: [PATCH 7/7] fix: cap broad search prefixes instead of failing --- CHANGELOG.md | 1 + src/internal/search/mod.rs | 123 ++++++++++++++++++++++++++++--------- 2 files changed, 94 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df8c506..dd75795 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ All notable changes to Craqle are documented here. so a fenced topic can be repaired on a running node. - Full-text search also matches indexed words that start with a query word of at least three characters, so `SAMD` finds `SAMD00012345`. Exact word matches rank above such prefix matches. + A very broad prefix keeps the prefix matches that fit the query budget instead of failing. ### Changed diff --git a/src/internal/search/mod.rs b/src/internal/search/mod.rs index 5703e85..e5290ba 100644 --- a/src/internal/search/mod.rs +++ b/src/internal/search/mod.rs @@ -3396,8 +3396,8 @@ impl SearchIndex { }) } - /// Documents holding a longer indexed word that starts with a long enough query word; none - /// without such a word. More documents than a quarter of the query budget are an error. + /// Documents holding a longer indexed word that starts with a long enough query word. Terms are + /// kept in word and dictionary order while their documents fit a quarter of the query budget. fn prefix_docs( &self, req: PrefixScan<'_, E>, @@ -3424,29 +3424,37 @@ impl SearchIndex { for reader in req.searcher.segment_readers() { let inverted = (reader.inverted_index(self.f_all_text)).map_err(SearchError::from)?; let terms = |word: &str| { - let mut range = inverted.terms().range().ge(word.as_bytes()); + let mut range = inverted.terms().range().gt(word.as_bytes()); if let Some(end) = prefix_end(word.as_bytes()) { range = range.lt(end); } range.into_stream().map_err(SearchError::from) }; - // The postings' document frequencies size the list once, so it never grows. - let mut size = 0usize; - for word in &words { + // Document frequencies size the list once, so it never grows; the first term whose + // documents pass the budget ends the expansion in this and every later segment. + let (mut size, mut kept, mut full) = (0usize, 0usize, false); + 'count: for word in &words { let mut terms = terms(word)?; while terms.advance() { (req.check)()?; - size = size.saturating_add(terms.value().doc_freq as usize); + let next = size.saturating_add(terms.value().doc_freq as usize); + let charge = next.saturating_mul(std::mem::size_of::()); + if bytes.saturating_add(charge) > limit { + full = true; + break 'count; + } + (size, kept) = (next, kept + 1); } } bytes = bytes.saturating_add(size.saturating_mul(std::mem::size_of::())); - if bytes > limit { - return Err(E::from(SearchError::ItemTooLarge { bytes, limit })); - } - let mut docs = Vec::with_capacity(size); - for word in &words { + let (mut docs, mut read) = (Vec::with_capacity(size), 0usize); + 'read: for word in &words { let mut terms = terms(word)?; while terms.advance() { + if read == kept { + break 'read; + } + read += 1; (req.check)()?; let option = IndexRecordOption::Basic; let mut postings = (inverted @@ -3467,6 +3475,9 @@ impl SearchIndex { docs.sort_unstable(); docs.dedup(); segments.insert(reader.segment_id(), Arc::new(docs)); + if full { + break; + } } Ok(Some(Box::new(PrefixDocs(Arc::new(segments))))) } @@ -4444,29 +4455,76 @@ mod tests { assert_eq!(hits[300].subject_iri, "urn:test:tier:prefix"); } - // An expansion over its budget fails like other oversized results, never as exact-only hits. + // A prefix over its budget keeps the terms that fit in dictionary order; exact hits stay first. #[test] - fn prefix_budget_fails() { + fn prefix_budget_caps() { let budget = MemoryBudget::new(1 << 30, 256 << 20).with_search_writer(20_000_000); let mut index = SearchIndex::memory_with_budget(budget).unwrap(); + for position in 0..2 { + let subject = format!("urn:test:cap:exact:{position}"); + index + .index_resource("urn:test:cap", &subject, Some("samd")) + .unwrap(); + } for position in 0..200 { let text = format!("samd{position:08}"); - let subject = format!("urn:test:budget:{position}"); + let subject = format!("urn:test:cap:{position}"); index - .index_resource("urn:test:budget", &subject, Some(&text)) + .index_resource("urn:test:cap", &subject, Some(&text)) .unwrap(); } index.commit().unwrap(); index.prepared_bytes = 12_796; - let result = index.search("samd", 10); + let hits = index.search("samd", 10).unwrap(); + let subjects = |hits: &[SearchHit]| -> Vec { + hits.iter().map(|hit| hit.subject_iri.clone()).collect() + }; + let again = index.search("samd", 10).unwrap(); + assert_eq!( + subjects(&hits), + subjects(&again), + "the same query keeps the same hits" + ); + let exact: Vec<_> = hits[..2] + .iter() + .map(|hit| hit.subject_iri.as_str()) + .collect(); assert!( - matches!(result, Err(SearchError::ItemTooLarge { .. })), - "{result:?}" + exact.iter().all(|subject| subject.contains(":exact:")), + "{exact:?}" + ); + assert!(hits[2..].iter().all(|hit| hit.score == 0.0)); + + let view = index.pin_view(); + let check = || Ok::<(), SearchError>(()); + let scan = PrefixScan { + searcher: &view.searcher, + query: "samd", + check: &check, + }; + let docs = index.prefix_docs(scan).unwrap().unwrap(); + let docs = docs.downcast_ref::().unwrap(); + let mut subjects = Vec::new(); + for (ord, reader) in view.searcher.segment_readers().iter().enumerate() { + for doc in docs + .0 + .get(&reader.segment_id()) + .map_or(&[][..], |docs| &docs[..]) + { + let address = DocAddress::new(ord as u32, *doc); + let stored = view.searcher.doc(address).unwrap(); + subjects.push(index.doc_to_hit(stored, 0.0).unwrap().subject_iri); + } + } + subjects.sort_by_key(|subject| subject.rsplit(':').next().unwrap().parse::().ok()); + let first: Vec<_> = (0..199).map(|n| format!("urn:test:cap:{n}")).collect(); + assert_eq!( + subjects, first, + "799 bytes keep the first 199 one-document terms" ); - assert_eq!(index.search("samd00000001", 10).unwrap().len(), 1); } - // The expansion charges its lists' exact capacity: 200 matches cost 800 bytes, no more. + // The cap charges the lists' exact capacity: 200 matches cost 800 bytes, no more. #[test] fn prefix_budget_exact() { let budget = MemoryBudget::new(1 << 30, 256 << 20).with_search_writer(20_000_000); @@ -4479,17 +4537,22 @@ mod tests { .unwrap(); } index.commit().unwrap(); + let kept = |index: &SearchIndex| { + let view = index.pin_view(); + let check = || Ok::<(), SearchError>(()); + let scan = PrefixScan { + searcher: &view.searcher, + query: "samd", + check: &check, + }; + let docs = index.prefix_docs(scan).unwrap().unwrap(); + let docs = docs.downcast_ref::().unwrap(); + docs.0.values().map(|docs| docs.len()).sum::() + }; index.prepared_bytes = 12_800; - assert_eq!(index.search("samd", 10).unwrap().len(), 10); + assert_eq!(kept(&index), 200, "800 bytes hold all 200 documents"); index.prepared_bytes = 12_796; - let result = index.search("samd", 10); - assert!(matches!( - result, - Err(SearchError::ItemTooLarge { - bytes: 800, - limit: 799 - }) - )); + assert_eq!(kept(&index), 199, "799 bytes hold one document less"); } // An exact match whose BM25 score rounds to zero still ranks above a prefix-only match.