diff --git a/src/bin/index.rs b/src/bin/index.rs index 186babb..4874099 100644 --- a/src/bin/index.rs +++ b/src/bin/index.rs @@ -4,6 +4,7 @@ use clap::Parser; use colored::Colorize; use semcode::indexer::{ list_shas_in_range, process_commits_pipeline, process_lore_commits_pipeline, KnownLoreIds, + LoreIndexIncomplete, }; use semcode::{measure, process_database_path, CodeVectorizer, DatabaseManager}; // Temporary call relationships are now embedded in function JSON columns @@ -688,8 +689,19 @@ async fn repack_lore_archive( } } +/// Rows a failed archive appended before it failed, counted as +/// LoreIndexResult::new_emails counts them for an archive that finished. +/// They stay in the table, so they count towards the compaction and FTS +/// update that new emails trigger. +fn appended_before_failure(e: &anyhow::Error) -> usize { + e.downcast_ref::() + .map_or(0, |incomplete| incomplete.appended) +} + /// Result of indexing a single lore archive struct LoreIndexResult { + /// Rows appended to the lore table. Emails already stored, such as + /// crossposts, and commits that hold no message are not counted. new_emails: usize, total_emails: usize, } @@ -774,7 +786,7 @@ async fn index_lore_archive( let new_count = new_commits.len(); // Process new commits - process_lore_commits_pipeline( + let new_emails = process_lore_commits_pipeline( archive_path, new_commits, db_manager.clone(), @@ -786,12 +798,12 @@ async fn index_lore_archive( .await?; println!( - "Indexed {} new emails from {} (total in archive: {})", - new_count, display_name, total_commits + "Indexed {} new emails from {} new commits of {} (total in archive: {})", + new_emails, new_count, display_name, total_commits ); Ok(LoreIndexResult { - new_emails: new_count, + new_emails, total_emails: total_commits, }) } @@ -1193,6 +1205,7 @@ async fn async_main() -> Result<()> { } Err(e) => { eprintln!("Error indexing {}: {}", lore_url, e); + total_new_emails += appended_before_failure(&e); failed_archives.push((lore_url.clone(), e.to_string())); continue; } @@ -1382,6 +1395,7 @@ async fn async_main() -> Result<()> { archives_processed += 1; } Err((name, e)) => { + total_new_emails += appended_before_failure(&e); failed_archives.push((name, e.to_string())); } } diff --git a/src/database/connection.rs b/src/database/connection.rs index cf3aa52..45c4b0a 100644 --- a/src/database/connection.rs +++ b/src/database/connection.rs @@ -161,6 +161,35 @@ pub enum LoreIdState { Pending, } +/// The emails of a batch that DatabaseManager::insert_lore_emails() did +/// not store, by index into the batch. Every other email of the batch is +/// in the lore table, either from this batch or from an earlier one. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct LoreInsertOutcome { + /// Emails whose append failed. Their ids are released, so inserting + /// them again tries the append again. + pub failed: Vec, + /// Emails whose id another inserter had pending. Insert them again + /// once that inserter has finished. + pub deferred: Vec, + /// Rows this call appended to the table. Emails already in the + /// table and copies of a message_id the batch holds more than once + /// are not counted. + pub appended: usize, +} + +impl LoreInsertOutcome { + /// Whether every email of the batch is stored. + pub fn is_complete(&self) -> bool { + self.failed.is_empty() && self.deferred.is_empty() + } + + /// The indices of every email not stored, failed or deferred. + pub fn not_stored(&self) -> impl Iterator + '_ { + self.failed.iter().chain(&self.deferred).copied() + } +} + pub struct DatabaseManager { connection: Connection, git_repo_path: String, @@ -6690,29 +6719,29 @@ impl DatabaseManager { } /// Insert lore emails into the database, returning the indices of - /// any emails that could not be stored. Callers must not record - /// commit SHAs for failed indices in the lore_indexed_commits - /// table, otherwise those emails are permanently lost. + /// the emails that were not stored. Callers must not record commit + /// SHAs for those indices in the lore_indexed_commits table, + /// otherwise those emails are permanently lost. /// /// `known_ids` holds every message_id already in the lore table /// (see get_lore_message_ids()) and is shared by all inserters. /// Emails whose message_id is stored are skipped. New ids are /// claimed as pending before the append, then marked stored, or /// released if their email failed. An email whose id another - /// inserter has pending is deferred: it is returned with the failed - /// indices, so its SHA is not recorded and the next run retries it - /// against the table. + /// inserter has pending is deferred: it is not stored and not + /// failed, and inserting it again once that inserter finishes either + /// skips it (the other copy was stored) or stores it. pub async fn insert_lore_emails( &self, emails: &[crate::types::LoreEmailInfo], known_ids: &std::sync::Mutex>, - ) -> Result> { + ) -> Result { use arrow::datatypes::{DataType, Field, Schema}; use std::collections::hash_map::Entry; use std::sync::Arc; if emails.is_empty() { - return Ok(Vec::new()); + return Ok(LoreInsertOutcome::default()); } tracing::info!( @@ -6810,7 +6839,11 @@ impl DatabaseManager { if new_indices.is_empty() { tracing::info!("insert_lore_emails: no new emails to insert"); - return Ok(deferred); + return Ok(LoreInsertOutcome { + failed: Vec::new(), + deferred: Self::with_duplicate_copies(emails, deferred), + appended: 0, + }); } // Use table.add() instead of merge_insert. merge_insert in @@ -6877,8 +6910,32 @@ impl DatabaseManager { tracing::info!("insert_lore_emails: Batch insertion complete"); - failed_indices.extend(deferred); - Ok(failed_indices) + let appended = new_indices.len() - failed_indices.len(); + Ok(LoreInsertOutcome { + failed: Self::with_duplicate_copies(emails, failed_indices), + deferred: Self::with_duplicate_copies(emails, deferred), + appended, + }) + } + + /// Extend `indices`, which name the one copy of each message_id that + /// insert_lore_emails() tried, to every email of the batch with that + /// message_id. The other copies are no more stored than the one + /// tried, and their commits must not be recorded as indexed. + fn with_duplicate_copies( + emails: &[crate::types::LoreEmailInfo], + indices: Vec, + ) -> Vec { + let ids: std::collections::HashSet<&str> = indices + .iter() + .map(|&i| emails[i].message_id.as_str()) + .collect(); + emails + .iter() + .enumerate() + .filter(|(_, e)| ids.contains(e.message_id.as_str())) + .map(|(i, _)| i) + .collect() } /// Return every message_id in the lore table, marked stored, for the diff --git a/src/database/mod.rs b/src/database/mod.rs index 3ab5eaa..2ed0480 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -21,7 +21,7 @@ mod symbol_filename; mod types; mod vectors; -pub use connection::{DatabaseManager, LoreIdState}; +pub use connection::{DatabaseManager, LoreIdState, LoreInsertOutcome}; use anyhow::Result; use arrow::array::RecordBatch; diff --git a/src/database/schema.rs b/src/database/schema.rs index d96c7cc..88905b5 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -2854,8 +2854,8 @@ mod tests { /// stored, and asserts that every email was stored. async fn insert_emails(db: &DatabaseManager, batch: &[LoreEmailInfo]) { let known_ids = std::sync::Mutex::new(db.get_lore_message_ids().await.unwrap()); - let failed = db.insert_lore_emails(batch, &known_ids).await.unwrap(); - assert!(failed.is_empty(), "emails not stored: {failed:?}"); + let outcome = db.insert_lore_emails(batch, &known_ids).await.unwrap(); + assert!(outcome.is_complete(), "emails not stored: {outcome:?}"); } /// Every BTree index on lore as (name, indexed rows, unindexed rows). diff --git a/src/git_range.rs b/src/git_range.rs index 9e6e649..af071a3 100644 --- a/src/git_range.rs +++ b/src/git_range.rs @@ -1,7 +1,9 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use crate::git::walk_tree_at_commit_with_repo; -use crate::indexer::{list_shas_in_range, process_commits_pipeline, recv_shared}; +use crate::indexer::{ + counted_problem, join_failed, list_shas_in_range, plural, process_commits_pipeline, recv_shared, +}; use crate::types::{ArgumentFunction, GlobalVariable}; use crate::{ DatabaseManager, DispatchSite, FunctionInfo, GitFileEntry, GitFileManifestEntry, @@ -139,27 +141,6 @@ struct GitTupleStats { failed_threads: usize, } -fn plural(n: usize, one: &str, many: &str) -> String { - format!("{n} {}", if n == 1 { one } else { many }) -} - -/// Join pipeline threads and count the ones that failed: returned an error -/// (a generator that could not list its files, a worker that stopped early) -/// or panicked. Either leaves files no counter saw, so any failure fails the -/// run. -fn join_failed(kind: &str, handles: Vec>>) -> usize { - let mut failed = 0; - for (id, handle) in handles.into_iter().enumerate() { - match handle.join() { - Ok(Ok(_)) => continue, - Ok(Err(e)) => tracing::error!("{kind} {id} failed: {e:#}"), - Err(panic) => tracing::error!("{kind} {id} thread panicked: {panic:?}"), - } - failed += 1; - } - failed -} - impl GitTupleStats { /// The run's error if any file was not read or stored. The run otherwise /// looks complete -- the branch would be marked current -- and only a @@ -174,19 +155,18 @@ impl GitTupleStats { /// What the run left undone, for the summary and the error, or `None` /// when it finished. fn incomplete_summary(&self) -> Option { - let mut problems = Vec::new(); - if self.failed_threads > 0 { - problems.push(format!( - "{} failed", - plural(self.failed_threads, "pipeline thread", "pipeline threads") - )); - } - if self.unread_files > 0 { - problems.push(format!( - "{} could not be read", - plural(self.unread_files, "file", "files") - )); - } + let mut problems: Vec<_> = [ + counted_problem( + self.failed_threads, + "pipeline thread", + "pipeline threads", + "failed", + ), + counted_problem(self.unread_files, "file", "files", "could not be read"), + ] + .into_iter() + .flatten() + .collect(); if self.failed_batches > 0 { problems.push(format!( "{} ({}) {} not stored", @@ -1334,8 +1314,8 @@ async fn process_git_tree_streaming(config: TreeStreamingConfig) -> Result( + kind: &str, + handles: impl IntoIterator>>, +) -> usize { + let mut failed = 0; + for (id, handle) in handles.into_iter().enumerate() { + match handle.join() { + Ok(Ok(_)) => continue, + Ok(Err(e)) => error!("{kind} {id} failed: {e:#}"), + Err(panic) => error!("{kind} {id} thread panicked: {panic:?}"), + } + failed += 1; + } + failed +} + /// Receive from a std channel shared by several inserter tasks without /// holding a tokio worker while waiting. /// @@ -426,20 +442,36 @@ fn parse_rfc2822_to_timestamp(date_str: &str) -> i64 { /// Parse email from a lore commit's 'm' file /// Extracts headers and body from an email message +/// +/// `Ok(None)` is a commit that holds no message: public-inbox records a +/// removed message as a commit with a 'd' file and no 'm'. Reading it +/// again gives the same answer. An error means the commit could not be +/// read -- a missing object, say -- and a later run may succeed. pub fn parse_email_from_commit( repo: &gix::Repository, commit_sha: &str, -) -> Result { +) -> Result> { // Resolve commit let commit = resolve_to_commit(repo, commit_sha)?; // Get the tree let tree = commit.tree()?; - // Find the 'm' file in the tree - let m_entry = tree - .find_entry("m") - .ok_or_else(|| anyhow::anyhow!("Commit {} does not contain file 'm'", commit_sha))?; + // Find the 'm' file in the tree. Tree::find_entry() drops decoding + // errors, which would make a malformed tree look like a removed + // message and get its commit recorded as indexed. + let mut m_entry = None; + for entry in tree.iter() { + let entry = + entry.with_context(|| format!("Failed to decode the tree of commit {commit_sha}"))?; + if entry.filename() == "m" { + m_entry = Some(entry); + break; + } + } + let Some(m_entry) = m_entry else { + return Ok(None); + }; // Read the file content let object = m_entry.object()?; @@ -504,7 +536,7 @@ pub fn parse_email_from_commit( // Parse date to Unix timestamp for efficient database filtering let date_timestamp = parse_rfc2822_to_timestamp(&headers.date); - Ok(crate::LoreEmailInfo { + Ok(Some(crate::LoreEmailInfo { git_commit_sha: commit_sha.to_string(), from: headers.from, date: headers.date, @@ -516,7 +548,7 @@ pub fn parse_email_from_commit( recipients, body, symbols, - }) + })) } /// Email header fields @@ -888,11 +920,172 @@ impl KnownLoreIds { } } +/// Pluralize `n` with `one` or `many`, for failure summaries. +pub(crate) fn plural(n: usize, one: &str, many: &str) -> String { + format!("{n} {}", if n == 1 { one } else { many }) +} + +pub(crate) fn counted_problem(count: usize, one: &str, many: &str, detail: &str) -> Option { + (count > 0).then(|| format!("{} {detail}", plural(count, one, many))) +} + +/// The error of a lore pipeline that finished without indexing every +/// commit it was given. +/// +/// Whatever the run did store stays stored and recorded. Every commit +/// counted here is left out of lore_indexed_commits, so the next run over +/// the archive reads it again. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct LoreIndexIncomplete { + /// Emails this run stored or found already stored. + pub stored: usize, + /// Of `stored`, the rows this run appended to the lore table. + pub appended: usize, + /// Parser, producer, and inserter threads that failed or panicked. + /// The commits such a thread held are counted nowhere else. + pub failed_threads: usize, + /// Commits whose message could not be read. + pub unread_commits: usize, + /// Emails that were read but could not be stored. + pub unstored_emails: usize, + /// Commits whose email was stored, or that hold no email, but that + /// could not be recorded as indexed. + pub unrecorded_commits: usize, +} + +impl LoreIndexIncomplete { + fn has_failures(&self) -> bool { + self.failed_threads > 0 + || self.unread_commits > 0 + || self.unstored_emails > 0 + || self.unrecorded_commits > 0 + } +} + +impl std::fmt::Display for LoreIndexIncomplete { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let problems: Vec<_> = [ + counted_problem( + self.failed_threads, + "pipeline thread", + "pipeline threads", + "failed", + ), + counted_problem(self.unread_commits, "email", "emails", "could not be read"), + counted_problem( + self.unstored_emails, + "email", + "emails", + "could not be stored", + ), + counted_problem( + self.unrecorded_commits, + "commit", + "commits", + "could not be recorded as indexed", + ), + ] + .into_iter() + .flatten() + .collect(); + write!( + f, + "lore indexing incomplete: {}; indexing again retries them ({} stored)", + problems.join(", "), + plural(self.stored, "email", "emails") + ) + } +} + +impl std::error::Error for LoreIndexIncomplete {} + +/// Failure counts the threads of one lore pipeline add to. +#[derive(Default)] +struct LoreFailureCounts { + unread_commits: AtomicUsize, + unstored_emails: AtomicUsize, + unrecorded_commits: AtomicUsize, +} + +/// Store one batch and record the commits of the emails now in the table. +/// Returns how many of the batch's emails are stored, how many rows were +/// appended for them, and the emails that were deferred because another +/// batch holds their message_id; failures are added to `counts`. +async fn store_lore_batch( + db: &DatabaseManager, + emails: Vec, + known_ids: &std::sync::Mutex>, + counts: &LoreFailureCounts, + inserter: &str, +) -> (usize, usize, Vec) { + let outcome = match db.insert_lore_emails(&emails, known_ids).await { + Ok(outcome) => outcome, + Err(e) => { + error!( + "{} failed to insert batch of {} emails: {:#}", + inserter, + emails.len(), + e + ); + counts + .unstored_emails + .fetch_add(emails.len(), Ordering::Relaxed); + return (0, 0, Vec::new()); + } + }; + counts + .unstored_emails + .fetch_add(outcome.failed.len(), Ordering::Relaxed); + + // Record commit SHAs only for emails that are in the table, so that + // the rest are read again on the next run. + let not_stored: HashSet = outcome.not_stored().collect(); + let shas: Vec = emails + .iter() + .enumerate() + .filter(|(i, _)| !not_stored.contains(i)) + .map(|(_, e)| e.git_commit_sha.as_str()) + .collect::>() + .into_iter() + .map(String::from) + .collect(); + if !shas.is_empty() { + if let Err(e) = db.insert_lore_indexed_commits(&shas).await { + error!( + "{} failed to record {} indexed commits: {:#}", + inserter, + shas.len(), + e + ); + counts + .unrecorded_commits + .fetch_add(shas.len(), Ordering::Relaxed); + } + } + + let stored = emails.len() - not_stored.len(); + let deferred_set: HashSet = outcome.deferred.into_iter().collect(); + let deferred = emails + .into_iter() + .enumerate() + .filter(|(i, _)| deferred_set.contains(i)) + .map(|(_, e)| e) + .collect(); + (stored, outcome.appended, deferred) +} + /// Process lore commits using a streaming pipeline /// Similar to process_commits_pipeline but parses email from 'm' file /// /// `known_ids` is the duplicate check for crossposts and partially failed /// runs; pass the same one for every archive of a run. +/// +/// Returns `Ok` with the rows appended to the lore table only when every +/// commit is recorded in lore_indexed_commits. Emails found already +/// stored, such as crossposts, and commits that hold no message add no +/// rows. Otherwise the error is a [`LoreIndexIncomplete`] counting what +/// was left undone, rows appended included; the emails that were stored +/// stay stored and recorded, and the rest are read again by the next run. pub async fn process_lore_commits_pipeline( repo_path: &std::path::Path, commit_shas: Vec, @@ -901,9 +1094,9 @@ pub async fn process_lore_commits_pipeline( num_workers: usize, num_inserters: usize, known_ids: &KnownLoreIds, -) -> Result<()> { +) -> Result { use indicatif::{ProgressBar, ProgressStyle}; - use std::sync::Mutex; + use std::sync::{Mutex, PoisonError}; let known_ids = known_ids.get(&db_manager).await?; @@ -927,6 +1120,11 @@ pub async fn process_lore_commits_pipeline( // Shared counters let processed_count = Arc::new(AtomicUsize::new(0)); let inserted_count = Arc::new(AtomicUsize::new(0)); + // Of those, the rows appended to the lore table. + let appended_count = Arc::new(AtomicUsize::new(0)); + let counts = Arc::new(LoreFailureCounts::default()); + // Commits that hold no message, recorded as indexed at the end. + let no_message_commits = Arc::new(Mutex::new(Vec::::new())); // Spawn progress updater thread let pb_clone = pb.clone(); @@ -940,82 +1138,90 @@ pub async fn process_lore_commits_pipeline( std::thread::sleep(std::time::Duration::from_millis(100)); }); - // Spawn worker threads to parse email metadata + // Spawn worker threads to parse email metadata. A commit that cannot + // be read is counted and the worker goes on; an error means the worker + // stopped, and the caller fails the run. The receiver lock guards + // nothing a panic can leave half-written, so a poisoned one is taken + // anyway: the panic already fails the run. let mut worker_handles = Vec::new(); for worker_id in 0..num_workers { let worker_commit_rx = shared_commit_rx.clone(); let worker_result_tx = result_tx.clone(); let worker_repo_path = repo_path.to_path_buf(); let worker_processed = processed_count.clone(); + let counts = counts.clone(); + let no_message_commits = no_message_commits.clone(); - let handle = thread::spawn(move || { + let handle = thread::spawn(move || -> Result<()> { // Open repository ONCE per worker thread - let thread_repo = match gix::discover(&worker_repo_path) { - Ok(r) => r, - Err(e) => { - tracing::error!("Worker {} failed to open repository: {}", worker_id, e); - return; - } + let thread_repo = gix::discover(&worker_repo_path).map_err(|e| { + anyhow::anyhow!( + "failed to open repository {}: {}", + worker_repo_path.display(), + e + ) + })?; + + let send = |batch: Vec| { + let len = batch.len(); + worker_result_tx.send(batch).map_err(|_| { + anyhow::anyhow!("every inserter stopped; {len} parsed emails were not sent") + }) }; let mut batch = Vec::new(); - loop { - // Get next commit SHA - let commit_sha = { - match worker_commit_rx.lock() { - Ok(rx) => rx.recv(), - Err(e) => { - tracing::error!("Worker {} failed to lock receiver: {}", worker_id, e); - break; + let commit_sha = worker_commit_rx + .lock() + .unwrap_or_else(PoisonError::into_inner) + .recv(); + // Channel closed: every commit has been handed out. + let Ok(sha) = commit_sha else { break }; + + match parse_email_from_commit(&thread_repo, &sha) { + Ok(Some(email_info)) => { + batch.push(email_info); + // Send batch when it reaches batch_size + if batch.len() >= batch_size { + send(std::mem::take(&mut batch))?; } } - }; - - match commit_sha { - Ok(sha) => { - match parse_email_from_commit(&thread_repo, &sha) { - Ok(email_info) => { - batch.push(email_info); - worker_processed.fetch_add(1, Ordering::Relaxed); - - // Send batch when it reaches batch_size - if batch.len() >= batch_size { - if worker_result_tx.send(batch.clone()).is_err() { - break; - } - batch.clear(); - } - } - Err(e) => { - warn!( - "Worker {} failed to parse email for {}: {}", - worker_id, sha, e - ); - worker_processed.fetch_add(1, Ordering::Relaxed); - } - } + Ok(None) => { + info!("Commit {} holds no message, recording it as indexed", sha); + no_message_commits + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(sha); } - Err(_) => { - // Channel closed, send remaining batch - if !batch.is_empty() { - let _ = worker_result_tx.send(batch); - } - break; + Err(e) => { + error!( + "Worker {} failed to read email for {}: {:#}", + worker_id, sha, e + ); + counts.unread_commits.fetch_add(1, Ordering::Relaxed); } } + worker_processed.fetch_add(1, Ordering::Relaxed); + } + if !batch.is_empty() { + send(batch)?; } + Ok(()) }); worker_handles.push(handle); } + // Only the workers hold the receiver now, so the producer stops if + // every worker has. + drop(shared_commit_rx); // Spawn producer thread to feed commit SHAs - let producer_handle = thread::spawn(move || { + let producer_handle = thread::spawn(move || -> Result<()> { for commit_sha in commit_shas { if commit_tx.send(commit_sha).is_err() { break; } } + Ok(()) }); // Close original senders @@ -1029,19 +1235,24 @@ pub async fn process_lore_commits_pipeline( let last_optimization_check = Arc::new(std::sync::Mutex::new(std::time::Instant::now())); let batches_inserted = Arc::new(AtomicUsize::new(0)); - // Spawn multiple database inserter tasks + // Spawn multiple database inserter tasks. Each returns the emails it + // deferred, to be stored once every inserter has finished. let mut inserter_handles = Vec::new(); for inserter_id in 0..num_inserters { let db_manager_clone = Arc::clone(&db_manager); let inserted_clone = inserted_count.clone(); + let appended_clone = appended_count.clone(); let pb_clone = pb.clone(); let result_rx_clone = shared_result_rx.clone(); let batches_counter = batches_inserted.clone(); let optimization_check_timer = last_optimization_check.clone(); let known_ids = known_ids.clone(); + let counts = counts.clone(); let handle = tokio::spawn(async move { + let name = format!("Inserter {inserter_id}"); + let mut deferred = Vec::new(); loop { // Get next batch from shared receiver. A parser only // sends a short batch when its input has run out, so @@ -1050,7 +1261,7 @@ pub async fn process_lore_commits_pipeline( // Otherwise each parser's last partial batch becomes // its own small append, dozens of tiny fragments per // archive on a refresh. - let batch = match recv_shared(&result_rx_clone, move |rx| { + let batch = recv_shared(&result_rx_clone, move |rx| { rx.recv().map(|mut emails| { while emails.len() < batch_size { match rx.recv() { @@ -1061,115 +1272,134 @@ pub async fn process_lore_commits_pipeline( emails }) }) - .await - { - Some(batch) => batch, - None => { - error!("Inserter {} failed to lock receiver", inserter_id); - break; - } + .await; + + let emails = match batch { + Some(Ok(emails)) => emails, + // Channel closed: every parser has finished. + Some(Err(_)) => break, + None => anyhow::bail!("{name} failed to lock receiver"), }; - match batch { - Ok(emails) => { - // Insert batch into database - match db_manager_clone - .insert_lore_emails(&emails, &known_ids) - .await - { - Err(e) => { - error!("Inserter {} failed to insert batch: {}", inserter_id, e); - } - Ok(failed_indices) => { - let failed_set: std::collections::HashSet = - failed_indices.into_iter().collect(); - let success_count = emails.len() - failed_set.len(); - - // Record commit SHAs only for emails that - // were actually stored, so that failed - // emails are retried on the next run. - let shas: Vec = emails - .iter() - .enumerate() - .filter(|(i, _)| !failed_set.contains(i)) - .map(|(_, e)| e.git_commit_sha.as_str()) - .collect::>() - .into_iter() - .map(String::from) - .collect(); - if !shas.is_empty() { - if let Err(e) = - db_manager_clone.insert_lore_indexed_commits(&shas).await - { - error!( - "Inserter {} failed to record indexed commits: {}", - inserter_id, e - ); - } - } - - let count = - inserted_clone.fetch_add(success_count, Ordering::Relaxed); - pb_clone.set_message(format!( - "Inserted {} emails", - count + success_count - )); - - // Track successful batch insertions and check for periodic optimization - let total_batches = - batches_counter.fetch_add(1, Ordering::Relaxed) + 1; - check_and_optimize_if_needed( - &db_manager_clone, - inserter_id, - total_batches, - &optimization_check_timer, - ) - .await; - } - } - } - Err(_) => { - // Channel closed - break; - } - } + let (stored, appended, mut batch_deferred) = + store_lore_batch(&db_manager_clone, emails, &known_ids, &counts, &name).await; + deferred.append(&mut batch_deferred); + appended_clone.fetch_add(appended, Ordering::Relaxed); + + let count = inserted_clone.fetch_add(stored, Ordering::Relaxed); + pb_clone.set_message(format!("Inserted {} emails", count + stored)); + + // Track batch insertions and check for periodic optimization + let total_batches = batches_counter.fetch_add(1, Ordering::Relaxed) + 1; + check_and_optimize_if_needed( + &db_manager_clone, + inserter_id, + total_batches, + &optimization_check_timer, + ) + .await; } + Ok::<_, anyhow::Error>(deferred) }); inserter_handles.push(handle); } + // Only the inserters hold the receiver now, so a parser's send fails, + // rather than blocking forever, if every inserter has stopped. + drop(shared_result_rx); + + let mut failed_threads = + join_failed("Worker", worker_handles) + join_failed("Producer", [producer_handle]); - // Wait for all workers to finish - for handle in worker_handles { - let _ = handle.join(); + // Wait for all inserters to finish + let mut deferred = Vec::new(); + for (id, handle) in inserter_handles.into_iter().enumerate() { + match handle.await { + Ok(Ok(mut emails)) => deferred.append(&mut emails), + Ok(Err(e)) => { + error!("Inserter {id} failed: {e:#}"); + failed_threads += 1; + } + Err(e) => { + error!("Inserter {id} failed: {e}"); + failed_threads += 1; + } + } } - // Wait for producer - let _ = producer_handle.join(); + // Every inserter has finished, so no batch holds an id pending (short + // of a panicked one). A deferred email's id is now stored, and the + // email is skipped and its commit recorded, or was released by a + // failed append, and the email is stored here. + if !deferred.is_empty() { + info!("Storing {} deferred emails", deferred.len()); + let (stored, appended, still_deferred) = + store_lore_batch(&db_manager, deferred, &known_ids, &counts, "Final pass").await; + inserted_count.fetch_add(stored, Ordering::Relaxed); + appended_count.fetch_add(appended, Ordering::Relaxed); + if !still_deferred.is_empty() { + error!( + "{} emails are still held by an inserter that did not finish", + still_deferred.len() + ); + counts + .unstored_emails + .fetch_add(still_deferred.len(), Ordering::Relaxed); + } + } - // Wait for all inserters to finish - for handle in inserter_handles { - let _ = handle.await; + let no_message_commits = std::mem::take( + &mut *no_message_commits + .lock() + .unwrap_or_else(PoisonError::into_inner), + ); + if let Err(e) = db_manager + .insert_lore_indexed_commits(&no_message_commits) + .await + { + error!( + "Failed to record {} commits without a message: {:#}", + no_message_commits.len(), + e + ); + counts + .unrecorded_commits + .fetch_add(no_message_commits.len(), Ordering::Relaxed); } + // A failed inserter can leave the ids of its batch pending; the + // emails it held are already counted as not stored. debug_assert!( - known_ids - .lock() - .unwrap_or_else(|e| e.into_inner()) - .values() - .all(|state| *state == LoreIdState::Stored), + failed_threads > 0 + || known_ids + .lock() + .unwrap_or_else(|e| e.into_inner()) + .values() + .all(|state| *state == LoreIdState::Stored), "lore message_id map holds pending ids after every inserter finished" ); + let outcome = LoreIndexIncomplete { + stored: inserted_count.load(Ordering::Relaxed), + appended: appended_count.load(Ordering::Relaxed), + failed_threads, + unread_commits: counts.unread_commits.load(Ordering::Relaxed), + unstored_emails: counts.unstored_emails.load(Ordering::Relaxed), + unrecorded_commits: counts.unrecorded_commits.load(Ordering::Relaxed), + }; + pb.finish_with_message(format!( "Processed {} emails, inserted {}", processed_count.load(Ordering::Relaxed), - inserted_count.load(Ordering::Relaxed) + outcome.stored )); // Wait for progress thread progress_thread.join().unwrap(); - Ok(()) + if outcome.has_failures() { + return Err(outcome.into()); + } + Ok(outcome.appended) } /// Index commits in a git range @@ -1522,14 +1752,14 @@ mod tests { async fn known_lore_ids_loads_once() { let (_dir, db) = lore_database().await; let side_map = std::sync::Mutex::new(std::collections::HashMap::new()); - let failed = db + let outcome = db .insert_lore_emails( &[lore_email("1111", ""), lore_email("2222", "")], &side_map, ) .await .unwrap(); - assert!(failed.is_empty()); + assert!(outcome.is_complete()); let known = KnownLoreIds::new(); let first = known.get(&db).await.unwrap(); @@ -1540,11 +1770,11 @@ mod tests { assert_eq!(ids.get(""), Some(&LoreIdState::Stored)); } - let failed = db + let outcome = db .insert_lore_emails(&[lore_email("3333", "")], &side_map) .await .unwrap(); - assert!(failed.is_empty()); + assert!(outcome.is_complete()); let second = known.get(&db).await.unwrap(); assert!(Arc::ptr_eq(&first, &second)); @@ -1556,7 +1786,7 @@ mod tests { } /// A batch holding an id that another batch has pending defers that - /// email: its index is returned as not stored, no row is written for + /// email: its index is returned as deferred, no row is written for /// it, and the other batch's claim is left alone. The rest of the /// batch is stored. #[tokio::test] @@ -1567,7 +1797,7 @@ mod tests { LoreIdState::Pending, )])); - let failed = db + let outcome = db .insert_lore_emails( &[ lore_email("4444", ""), @@ -1578,7 +1808,14 @@ mod tests { .await .unwrap(); - assert_eq!(failed, vec![0]); + assert_eq!( + outcome, + crate::LoreInsertOutcome { + failed: Vec::new(), + deferred: vec![0], + appended: 1, + } + ); assert_eq!( lore_rows(&db, Some("message_id = ''".to_string())).await, 0 @@ -1592,6 +1829,41 @@ mod tests { assert_eq!(map.get(""), Some(&LoreIdState::Stored)); } + /// Every copy of a deferred message_id in the batch is deferred, not + /// only the one copy the batch tried, so that no copy's commit is + /// recorded as indexed while its message is not stored. + #[tokio::test] + async fn every_copy_of_a_pending_id_is_deferred() { + let (_dir, db) = lore_database().await; + let map = std::sync::Mutex::new(HashMap::from([( + "".to_string(), + LoreIdState::Pending, + )])); + + let outcome = db + .insert_lore_emails( + &[ + lore_email("1111", ""), + lore_email("2222", ""), + lore_email("3333", ""), + lore_email("4444", ""), + ], + &map, + ) + .await + .unwrap(); + + assert_eq!( + outcome, + crate::LoreInsertOutcome { + failed: Vec::new(), + deferred: vec![0, 2], + appended: 1, + } + ); + assert_eq!(lore_rows(&db, None).await, 1); + } + /// A failed scan leaves the map unloaded: the next get() scans again. #[tokio::test] async fn known_lore_ids_retries_after_failed_load() { @@ -1609,11 +1881,11 @@ mod tests { db.create_tables().await.unwrap(); let side_map = std::sync::Mutex::new(HashMap::new()); - let failed = db + let outcome = db .insert_lore_emails(&[lore_email("6666", "")], &side_map) .await .unwrap(); - assert!(failed.is_empty()); + assert!(outcome.is_complete()); let ids = known.get(&db).await.unwrap(); assert_eq!( @@ -1676,7 +1948,7 @@ mod tests { ) .await .unwrap(); - process_lore_commits_pipeline( + let appended = process_lore_commits_pipeline( archive_b.path(), vec![cross_b.clone(), only_b.clone()], db.clone(), @@ -1687,6 +1959,7 @@ mod tests { ) .await .unwrap(); + assert_eq!(appended, 1, "the crosspost was counted as a new row"); assert_eq!( lore_rows(&db, Some("message_id = ''".to_string())).await, @@ -1699,4 +1972,337 @@ mod tests { assert!(indexed.contains(sha), "{sha} not recorded as indexed"); } } + + /// A lore archive of one commit per message id, oldest first. + fn lore_archive(message_ids: &[&str]) -> (tempfile::TempDir, Vec) { + let archive = tempfile::tempdir().unwrap(); + git(archive.path(), &["init", "-q", "-b", "master"]); + let shas = message_ids + .iter() + .map(|id| commit_lore_message(archive.path(), id, id)) + .collect(); + (archive, shas) + } + + /// Run the pipeline as `semcode-index --lore` does: a fresh duplicate + /// check, two parsers and two inserters, over the commits not yet + /// recorded as indexed. Fails rather than hangs if the pipeline does + /// not finish. + async fn index_lore_run( + db: &Arc, + archive: &std::path::Path, + commits: &[String], + ) -> Result { + let indexed = db.get_indexed_lore_commits().await.unwrap(); + let pending = commits + .iter() + .filter(|sha| !indexed.contains(*sha)) + .cloned() + .collect(); + index_lore_commits(db, archive, pending, &KnownLoreIds::new()).await + } + + async fn index_lore_commits( + db: &Arc, + archive: &std::path::Path, + commits: Vec, + known: &KnownLoreIds, + ) -> Result { + tokio::time::timeout( + std::time::Duration::from_secs(120), + process_lore_commits_pipeline(archive, commits, db.clone(), 1, 2, 2, known), + ) + .await + .expect("lore pipeline did not finish") + } + + fn incomplete(result: Result) -> LoreIndexIncomplete { + let e = result.expect_err("a run that left commits unindexed reported success"); + e.downcast_ref::() + .unwrap_or_else(|| panic!("not a LoreIndexIncomplete: {e:#}")) + .clone() + } + + async fn assert_all_recorded(db: &DatabaseManager, commits: &[String]) { + let indexed = db.get_indexed_lore_commits().await.unwrap(); + for sha in commits { + assert!(indexed.contains(sha), "{sha} not recorded as indexed"); + } + } + + /// A commit whose message cannot be read fails the run. The emails + /// read beside it stay stored and recorded, and the unread commit is + /// left unrecorded, so the next run reads it and completes the + /// archive. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn unreadable_email_fails_the_run_and_is_retried() { + let (_dir, db) = lore_database().await; + let (archive, commits) = lore_archive(&["", "", ""]); + + // Delete the loose blob of the middle commit's message. + let blob = git(archive.path(), &["rev-parse", &format!("{}:m", commits[1])]); + let blob_path = archive + .path() + .join(".git/objects") + .join(&blob[..2]) + .join(&blob[2..]); + let blob_bytes = std::fs::read(&blob_path).unwrap(); + std::fs::remove_file(&blob_path).unwrap(); + + let outcome = incomplete(index_lore_run(&db, archive.path(), &commits).await); + assert_eq!( + outcome, + LoreIndexIncomplete { + stored: 2, + appended: 2, + unread_commits: 1, + ..Default::default() + } + ); + let message = outcome.to_string(); + assert!( + message.contains("1 email could not be read"), + "unexpected summary: {message}" + ); + assert_eq!(lore_rows(&db, None).await, 2); + let indexed = db.get_indexed_lore_commits().await.unwrap(); + assert!(indexed.contains(&commits[0])); + assert!(!indexed.contains(&commits[1])); + assert!(indexed.contains(&commits[2])); + + std::fs::write(&blob_path, blob_bytes).unwrap(); + assert_eq!( + index_lore_run(&db, archive.path(), &commits).await.unwrap(), + 1 + ); + assert_eq!(lore_rows(&db, None).await, 3); + assert_all_recorded(&db, &commits).await; + } + + /// Emails that cannot be stored fail the run and leave their commits + /// unrecorded, so the next run stores them. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn unstored_emails_fail_the_run_and_are_retried() { + let (_dir, db) = lore_database().await; + let (archive, commits) = lore_archive(&["", ""]); + + // Load the duplicate check, then take the table away from the + // inserters. + let known = KnownLoreIds::new(); + known.get(&db).await.unwrap(); + db.connection().drop_table("lore", &[]).await.unwrap(); + + let outcome = + incomplete(index_lore_commits(&db, archive.path(), commits.clone(), &known).await); + assert_eq!( + outcome, + LoreIndexIncomplete { + unstored_emails: 2, + ..Default::default() + } + ); + assert!(db.get_indexed_lore_commits().await.unwrap().is_empty()); + + db.create_tables().await.unwrap(); + assert_eq!( + index_lore_run(&db, archive.path(), &commits).await.unwrap(), + 2 + ); + assert_eq!(lore_rows(&db, None).await, 2); + assert_all_recorded(&db, &commits).await; + } + + /// A failed run that found its only readable email already stored + /// appended nothing, so it reports no new rows for maintenance. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn already_stored_email_is_not_counted_as_appended() { + let (_dir, db) = lore_database().await; + let (archive, commits) = lore_archive(&["", ""]); + + // Store as another archive's crosspost would, without + // recording this archive's commit. + let side_map = std::sync::Mutex::new(HashMap::new()); + db.insert_lore_emails(&[lore_email("9999", "")], &side_map) + .await + .unwrap(); + + let blob = git(archive.path(), &["rev-parse", &format!("{}:m", commits[1])]); + std::fs::remove_file( + archive + .path() + .join(".git/objects") + .join(&blob[..2]) + .join(&blob[2..]), + ) + .unwrap(); + + let outcome = incomplete(index_lore_run(&db, archive.path(), &commits).await); + assert_eq!( + outcome, + LoreIndexIncomplete { + stored: 1, + appended: 0, + unread_commits: 1, + ..Default::default() + } + ); + assert_eq!(lore_rows(&db, None).await, 1); + } + + /// A commit whose tree cannot be decoded is unread, not taken for a + /// removed message: it fails the run and is left unrecorded. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn malformed_tree_is_not_recorded() { + let (_dir, db) = lore_database().await; + let (archive, mut commits) = lore_archive(&[""]); + + let bad_tree = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(bad_tree.path(), b"100644 no-nul-or-object-id").unwrap(); + let tree = git( + archive.path(), + &[ + "hash-object", + "-t", + "tree", + "--literally", + "-w", + bad_tree.path().to_str().unwrap(), + ], + ); + let commit = git( + archive.path(), + &["commit-tree", &tree, "-p", &commits[0], "-m", "bad tree"], + ); + commits.push(commit.clone()); + + let outcome = incomplete(index_lore_run(&db, archive.path(), &commits).await); + assert_eq!( + outcome, + LoreIndexIncomplete { + stored: 1, + appended: 1, + unread_commits: 1, + ..Default::default() + } + ); + let indexed = db.get_indexed_lore_commits().await.unwrap(); + assert!(indexed.contains(&commits[0])); + assert!(!indexed.contains(&commit)); + } + + /// Stored emails whose commits cannot be recorded fail the run. The + /// next run reads them again, finds them stored, and records them + /// without storing a second copy. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn unrecorded_commits_fail_the_run_and_are_retried() { + let (_dir, db) = lore_database().await; + let (archive, commits) = lore_archive(&["", ""]); + db.connection() + .drop_table("lore_indexed_commits", &[]) + .await + .unwrap(); + + let outcome = incomplete(index_lore_run(&db, archive.path(), &commits).await); + assert_eq!( + outcome, + LoreIndexIncomplete { + stored: 2, + appended: 2, + unrecorded_commits: 2, + ..Default::default() + } + ); + assert_eq!(lore_rows(&db, None).await, 2); + + db.create_tables().await.unwrap(); + assert_eq!( + index_lore_run(&db, archive.path(), &commits).await.unwrap(), + 0 + ); + assert_eq!(lore_rows(&db, None).await, 2); + assert_all_recorded(&db, &commits).await; + } + + /// Parsers that cannot open the archive fail the run instead of + /// leaving every commit unread and reporting success. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn failed_parsers_fail_the_run() { + let (_dir, db) = lore_database().await; + let (_archive, commits) = lore_archive(&[""]); + let not_a_repo = tempfile::tempdir().unwrap(); + + let outcome = incomplete(index_lore_run(&db, not_a_repo.path(), &commits).await); + assert_eq!( + outcome, + LoreIndexIncomplete { + failed_threads: 2, + ..Default::default() + } + ); + assert!(db.get_indexed_lore_commits().await.unwrap().is_empty()); + } + + /// public-inbox records a removed message as a commit with a 'd' file + /// and no 'm'. Such a commit holds nothing to store: it does not fail + /// the run and is recorded, so later runs do not read it again. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn commit_without_a_message_is_recorded() { + let (_dir, db) = lore_database().await; + let (archive, mut commits) = lore_archive(&[""]); + git(archive.path(), &["rm", "-q", "m"]); + std::fs::write(archive.path().join("d"), "removed\n").unwrap(); + git(archive.path(), &["add", "d"]); + git(archive.path(), &["commit", "-q", "-m", "remove"]); + commits.push(git(archive.path(), &["rev-parse", "HEAD"])); + + assert_eq!( + index_lore_run(&db, archive.path(), &commits).await.unwrap(), + 1 + ); + assert_eq!(lore_rows(&db, None).await, 1); + assert_all_recorded(&db, &commits).await; + } + + /// One message id in several commits of an archive, spread over + /// batches that two inserters store at once. Whichever batch defers + /// the id to another, the run stores one copy and records every + /// commit. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn duplicate_message_ids_are_stored_once_and_recorded() { + let (_dir, db) = lore_database().await; + let (archive, commits) = lore_archive(&[ + "", "", "", "", "", "", "", + ]); + + assert_eq!( + index_lore_run(&db, archive.path(), &commits).await.unwrap(), + 4 + ); + assert_eq!( + lore_rows(&db, Some("message_id = ''".to_string())).await, + 1 + ); + assert_eq!(lore_rows(&db, None).await, 4); + assert_all_recorded(&db, &commits).await; + } + + /// The summary names each kind of failure, and the stored count. + #[test] + fn incomplete_summary_lists_each_failure() { + let outcome = LoreIndexIncomplete { + stored: 900, + appended: 800, + failed_threads: 1, + unread_commits: 2, + unstored_emails: 100, + unrecorded_commits: 1, + }; + assert_eq!( + outcome.to_string(), + "lore indexing incomplete: 1 pipeline thread failed, \ + 2 emails could not be read, 100 emails could not be stored, \ + 1 commit could not be recorded as indexed; \ + indexing again retries them (900 emails stored)" + ); + } } diff --git a/src/lib.rs b/src/lib.rs index 1fadc26..ff62518 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,7 @@ pub mod search; pub use database::processed_files::ProcessedFileRecord; pub use database::resolution::{Evidence, IndirectCaller}; pub use database::SCHEMA_VERSION; -pub use database::{DatabaseManager, LoreIdState}; +pub use database::{DatabaseManager, LoreIdState, LoreInsertOutcome}; pub use database_utils::process_database_path; pub use git::{get_git_sha, get_git_sha_for_workdir}; pub use hash::{compute_content_hash, compute_file_hash}; diff --git a/tests/lore_symbol_search.rs b/tests/lore_symbol_search.rs index 00fac9d..83749ba 100644 --- a/tests/lore_symbol_search.rs +++ b/tests/lore_symbol_search.rs @@ -22,8 +22,8 @@ async fn database() -> (tempfile::TempDir, DatabaseManager) { async fn insert(db: &DatabaseManager, emails: &[LoreEmailInfo]) { let known_ids = std::sync::Mutex::new(db.get_lore_message_ids().await.unwrap()); - let failed = db.insert_lore_emails(emails, &known_ids).await.unwrap(); - assert!(failed.is_empty(), "emails not stored: {failed:?}"); + let outcome = db.insert_lore_emails(emails, &known_ids).await.unwrap(); + assert!(outcome.is_complete(), "emails not stored: {outcome:?}"); db.create_lore_fts_indices().await.unwrap(); } diff --git a/tests/lore_thread.rs b/tests/lore_thread.rs index a4a2154..8a40d95 100644 --- a/tests/lore_thread.rs +++ b/tests/lore_thread.rs @@ -27,8 +27,8 @@ async fn database() -> (tempfile::TempDir, DatabaseManager) { async fn insert(db: &DatabaseManager, emails: &[LoreEmailInfo]) { let known_ids = std::sync::Mutex::new(db.get_lore_message_ids().await.unwrap()); - let failed = db.insert_lore_emails(emails, &known_ids).await.unwrap(); - assert!(failed.is_empty(), "emails not stored: {failed:?}"); + let outcome = db.insert_lore_emails(emails, &known_ids).await.unwrap(); + assert!(outcome.is_complete(), "emails not stored: {outcome:?}"); } fn email(message_id: &str, in_reply_to: Option<&str>, references: Option<&str>) -> LoreEmailInfo {