Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions src/bin/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use clap::Parser;
use colored::Colorize;
use semcode::indexer::{
list_shas_in_range, process_commits_pipeline, process_lore_commits_pipeline, KnownLoreIds,
LoreIndexIncomplete,
};
use semcode::{measure, process_database_path, CodeVectorizer, DatabaseManager};
// Temporary call relationships are now embedded in function JSON columns
Expand Down Expand Up @@ -688,8 +689,19 @@ async fn repack_lore_archive(
}
}

/// Rows a failed archive appended before it failed, counted as
/// LoreIndexResult::new_emails counts them for an archive that finished.
/// They stay in the table, so they count towards the compaction and FTS
/// update that new emails trigger.
fn appended_before_failure(e: &anyhow::Error) -> usize {
e.downcast_ref::<LoreIndexIncomplete>()
.map_or(0, |incomplete| incomplete.appended)
}

/// Result of indexing a single lore archive
struct LoreIndexResult {
/// Rows appended to the lore table. Emails already stored, such as
/// crossposts, and commits that hold no message are not counted.
new_emails: usize,
total_emails: usize,
}
Expand Down Expand Up @@ -774,7 +786,7 @@ async fn index_lore_archive(
let new_count = new_commits.len();

// Process new commits
process_lore_commits_pipeline(
let new_emails = process_lore_commits_pipeline(
archive_path,
new_commits,
db_manager.clone(),
Expand All @@ -786,12 +798,12 @@ async fn index_lore_archive(
.await?;

println!(
"Indexed {} new emails from {} (total in archive: {})",
new_count, display_name, total_commits
"Indexed {} new emails from {} new commits of {} (total in archive: {})",
new_emails, new_count, display_name, total_commits
);

Ok(LoreIndexResult {
new_emails: new_count,
new_emails,
total_emails: total_commits,
})
}
Expand Down Expand Up @@ -1193,6 +1205,7 @@ async fn async_main() -> Result<()> {
}
Err(e) => {
eprintln!("Error indexing {}: {}", lore_url, e);
total_new_emails += appended_before_failure(&e);
failed_archives.push((lore_url.clone(), e.to_string()));
continue;
}
Expand Down Expand Up @@ -1382,6 +1395,7 @@ async fn async_main() -> Result<()> {
archives_processed += 1;
}
Err((name, e)) => {
total_new_emails += appended_before_failure(&e);
failed_archives.push((name, e.to_string()));
}
}
Expand Down
79 changes: 68 additions & 11 deletions src/database/connection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,35 @@ pub enum LoreIdState {
Pending,
}

/// The emails of a batch that DatabaseManager::insert_lore_emails() did
/// not store, by index into the batch. Every other email of the batch is
/// in the lore table, either from this batch or from an earlier one.
#[derive(Debug, Default, PartialEq, Eq)]
pub struct LoreInsertOutcome {
/// Emails whose append failed. Their ids are released, so inserting
/// them again tries the append again.
pub failed: Vec<usize>,
/// Emails whose id another inserter had pending. Insert them again
/// once that inserter has finished.
pub deferred: Vec<usize>,
/// Rows this call appended to the table. Emails already in the
/// table and copies of a message_id the batch holds more than once
/// are not counted.
pub appended: usize,
}

impl LoreInsertOutcome {
/// Whether every email of the batch is stored.
pub fn is_complete(&self) -> bool {
self.failed.is_empty() && self.deferred.is_empty()
}

/// The indices of every email not stored, failed or deferred.
pub fn not_stored(&self) -> impl Iterator<Item = usize> + '_ {
self.failed.iter().chain(&self.deferred).copied()
}
}

pub struct DatabaseManager {
connection: Connection,
git_repo_path: String,
Expand Down Expand Up @@ -6690,29 +6719,29 @@ impl DatabaseManager {
}

/// Insert lore emails into the database, returning the indices of
/// any emails that could not be stored. Callers must not record
/// commit SHAs for failed indices in the lore_indexed_commits
/// table, otherwise those emails are permanently lost.
/// the emails that were not stored. Callers must not record commit
/// SHAs for those indices in the lore_indexed_commits table,
/// otherwise those emails are permanently lost.
///
/// `known_ids` holds every message_id already in the lore table
/// (see get_lore_message_ids()) and is shared by all inserters.
/// Emails whose message_id is stored are skipped. New ids are
/// claimed as pending before the append, then marked stored, or
/// released if their email failed. An email whose id another
/// inserter has pending is deferred: it is returned with the failed
/// indices, so its SHA is not recorded and the next run retries it
/// against the table.
/// inserter has pending is deferred: it is not stored and not
/// failed, and inserting it again once that inserter finishes either
/// skips it (the other copy was stored) or stores it.
pub async fn insert_lore_emails(
&self,
emails: &[crate::types::LoreEmailInfo],
known_ids: &std::sync::Mutex<HashMap<String, LoreIdState>>,
) -> Result<Vec<usize>> {
) -> Result<LoreInsertOutcome> {
use arrow::datatypes::{DataType, Field, Schema};
use std::collections::hash_map::Entry;
use std::sync::Arc;

if emails.is_empty() {
return Ok(Vec::new());
return Ok(LoreInsertOutcome::default());
}

tracing::info!(
Expand Down Expand Up @@ -6810,7 +6839,11 @@ impl DatabaseManager {

if new_indices.is_empty() {
tracing::info!("insert_lore_emails: no new emails to insert");
return Ok(deferred);
return Ok(LoreInsertOutcome {
failed: Vec::new(),
deferred: Self::with_duplicate_copies(emails, deferred),
appended: 0,
});
}

// Use table.add() instead of merge_insert. merge_insert in
Expand Down Expand Up @@ -6877,8 +6910,32 @@ impl DatabaseManager {

tracing::info!("insert_lore_emails: Batch insertion complete");

failed_indices.extend(deferred);
Ok(failed_indices)
let appended = new_indices.len() - failed_indices.len();
Ok(LoreInsertOutcome {
failed: Self::with_duplicate_copies(emails, failed_indices),
deferred: Self::with_duplicate_copies(emails, deferred),
appended,
})
}

/// Extend `indices`, which name the one copy of each message_id that
/// insert_lore_emails() tried, to every email of the batch with that
/// message_id. The other copies are no more stored than the one
/// tried, and their commits must not be recorded as indexed.
fn with_duplicate_copies(
emails: &[crate::types::LoreEmailInfo],
indices: Vec<usize>,
) -> Vec<usize> {
let ids: std::collections::HashSet<&str> = indices
.iter()
.map(|&i| emails[i].message_id.as_str())
.collect();
emails
.iter()
.enumerate()
.filter(|(_, e)| ids.contains(e.message_id.as_str()))
.map(|(i, _)| i)
.collect()
}

/// Return every message_id in the lore table, marked stored, for the
Expand Down
2 changes: 1 addition & 1 deletion src/database/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ mod symbol_filename;
mod types;
mod vectors;

pub use connection::{DatabaseManager, LoreIdState};
pub use connection::{DatabaseManager, LoreIdState, LoreInsertOutcome};

use anyhow::Result;
use arrow::array::RecordBatch;
Expand Down
4 changes: 2 additions & 2 deletions src/database/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2854,8 +2854,8 @@ mod tests {
/// stored, and asserts that every email was stored.
async fn insert_emails(db: &DatabaseManager, batch: &[LoreEmailInfo]) {
let known_ids = std::sync::Mutex::new(db.get_lore_message_ids().await.unwrap());
let failed = db.insert_lore_emails(batch, &known_ids).await.unwrap();
assert!(failed.is_empty(), "emails not stored: {failed:?}");
let outcome = db.insert_lore_emails(batch, &known_ids).await.unwrap();
assert!(outcome.is_complete(), "emails not stored: {outcome:?}");
}

/// Every BTree index on lore as (name, indexed rows, unindexed rows).
Expand Down
54 changes: 17 additions & 37 deletions src/git_range.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
// SPDX-License-Identifier: MIT OR Apache-2.0

use crate::git::walk_tree_at_commit_with_repo;
use crate::indexer::{list_shas_in_range, process_commits_pipeline, recv_shared};
use crate::indexer::{
counted_problem, join_failed, list_shas_in_range, plural, process_commits_pipeline, recv_shared,
};
use crate::types::{ArgumentFunction, GlobalVariable};
use crate::{
DatabaseManager, DispatchSite, FunctionInfo, GitFileEntry, GitFileManifestEntry,
Expand Down Expand Up @@ -139,27 +141,6 @@ struct GitTupleStats {
failed_threads: usize,
}

fn plural(n: usize, one: &str, many: &str) -> String {
format!("{n} {}", if n == 1 { one } else { many })
}

/// Join pipeline threads and count the ones that failed: returned an error
/// (a generator that could not list its files, a worker that stopped early)
/// or panicked. Either leaves files no counter saw, so any failure fails the
/// run.
fn join_failed<T>(kind: &str, handles: Vec<thread::JoinHandle<Result<T>>>) -> usize {
let mut failed = 0;
for (id, handle) in handles.into_iter().enumerate() {
match handle.join() {
Ok(Ok(_)) => continue,
Ok(Err(e)) => tracing::error!("{kind} {id} failed: {e:#}"),
Err(panic) => tracing::error!("{kind} {id} thread panicked: {panic:?}"),
}
failed += 1;
}
failed
}

impl GitTupleStats {
/// The run's error if any file was not read or stored. The run otherwise
/// looks complete -- the branch would be marked current -- and only a
Expand All @@ -174,19 +155,18 @@ impl GitTupleStats {
/// What the run left undone, for the summary and the error, or `None`
/// when it finished.
fn incomplete_summary(&self) -> Option<String> {
let mut problems = Vec::new();
if self.failed_threads > 0 {
problems.push(format!(
"{} failed",
plural(self.failed_threads, "pipeline thread", "pipeline threads")
));
}
if self.unread_files > 0 {
problems.push(format!(
"{} could not be read",
plural(self.unread_files, "file", "files")
));
}
let mut problems: Vec<_> = [
counted_problem(
self.failed_threads,
"pipeline thread",
"pipeline threads",
"failed",
),
counted_problem(self.unread_files, "file", "files", "could not be read"),
]
.into_iter()
.flatten()
.collect();
if self.failed_batches > 0 {
problems.push(format!(
"{} ({}) {} not stored",
Expand Down Expand Up @@ -1334,8 +1314,8 @@ async fn process_git_tree_streaming(config: TreeStreamingConfig) -> Result<GitTu
}

// Wait for the generator, then all workers, to complete
let failed_threads = join_failed("Tree generator", vec![generator_handle])
+ join_failed("Worker", worker_handles);
let failed_threads =
join_failed("Tree generator", [generator_handle]) + join_failed("Worker", worker_handles);

// Wait for all inserter tasks to finish
for (inserter_id, (handle, in_flight_files)) in inserter_handles.into_iter().enumerate() {
Expand Down
Loading
Loading