diff --git a/README.md b/README.md index 782300c..5d55ce2 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,52 @@ if !result.all_correct() && result.repair_possible { } ``` +### In-memory repair (no file I/O) + +`Repair` rebuilds missing slices from slices you already hold in memory, such +as article bodies from a Usenet download, plus recovery blocks. It reads and +writes no files. Feed each present slice once, in any order, then solve: + +```rust +use rust_par2::recovery::RecoveryBlock; +use rust_par2::{BlockMap, Repair}; +# fn demo(file_set: &rust_par2::Par2FileSet, recovery: Vec, +# present: Vec<(usize, Vec)>, mut missing: Vec) -> Result<(), rust_par2::RepairError> { +// Global slice indices run across files in the Main packet's file order. +let map = BlockMap::new(file_set); +let slice_size = file_set.slice_size as usize; + +let mut repair = Repair::new_owning(recovery, slice_size, map.total_blocks as usize)?; + +for (index, bytes) in &present { + // Verify before feeding: a corrupt slice cannot be taken back out. + let loc = map.global_to_file(*index).unwrap(); + let file = &file_set.files[&loc.file.file_id]; + if file.slice_matches(loc.local_index as usize, bytes, file_set.slice_size) { + repair.feed(*index, bytes)?; // the last slice of a file may be short + } else { + missing.push(*index); + } +} +repair.truncate_rows(missing.len()); // optional: free rows solve won't use + +// Every slice must be either fed or listed as missing. +for (index, slice) in repair.solve(&missing)? { + let loc = map.global_to_file(index).unwrap(); + // `slice` is zero-padded to slice_size; write only loc.len bytes at loc.offset. + let _ = (&loc.file.filename, loc.offset, &slice[..loc.len]); +} +# Ok(()) } +``` + +`solve` fails with `RepairError::InsufficientRecovery` when more slices are +missing than recovery rows are loaded. Peak memory is `rows × slice_size` for +the accumulators plus `missing × slice_size` for the output, plus whatever you +buffer before feeding. `feed_batch` takes several slices at once and uses the +same cache-tiled SIMD path as the file-based repair, which is built on `Repair`. +`SliceHasher` computes the IFSC MD5 + CRC32 pair incrementally if a slice +arrives in pieces. + ## Performance Benchmarked against [par2cmdline-turbo](https://github.com/animetosho/par2cmdline-turbo) @@ -221,6 +267,9 @@ than treating the entire file as lost. | `repair(file_set, dir)` | Verify + repair in one call | | `repair_from_verify(file_set, dir, verify_result)` | Repair with pre-computed verify | | `compute_hash_16k(path)` | MD5 of first 16 KiB (for file identification) | +| `Repair::new_owning(recovery, slice_size, total)` | In-memory repair builder: `feed`, `feed_batch`, `solve` | +| `BlockMap::new(file_set)` | Map global slice indices to (file, offset, length) | +| `SliceHasher::checksum(data, slice_size)` | IFSC MD5 + CRC32 of a slice (zero-padded) | ### Key types @@ -243,6 +292,8 @@ than treating the entire file as lost. | `RepairError::SingularMatrix` | Recovery blocks are linearly dependent | | `RepairError::NoDamage` | Nothing to repair (all files intact) | | `RepairError::VerifyFailed` | Post-repair verification failed | +| `RepairError::SliceOutOfRange` / `SliceTooLong` / `DuplicateSlice` / `IncompleteFeed` | Invalid input to the `Repair` builder | +| `RepairError::InvalidSliceSize` / `InvalidRecoveryBlock` | Bad `Repair` construction arguments | ## Dependencies diff --git a/src/lib.rs b/src/lib.rs index d960be9..b4e2fca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,13 +33,16 @@ pub mod matrix; mod packets; pub mod recovery; pub mod repair; +pub mod slice; pub mod types; mod verify; pub use packets::{ParseError, parse_par2_file as parse, parse_par2_reader}; pub use repair::{ - RepairError, RepairResult, repair, repair_from_verify, repair_from_verify_no_reverify, + BlockFile, BlockMap, Repair, RepairError, RepairResult, SliceLocation, build_block_map, repair, + repair_from_verify, repair_from_verify_no_reverify, }; +pub use slice::SliceHasher; pub use types::{ DamagedFile, Md5Hash, MissingFile, Par2File, Par2FileSet, SliceChecksum, VerifiedFile, VerifyResult, diff --git a/src/recovery.rs b/src/recovery.rs index ddef4ae..fb08b3c 100644 --- a/src/recovery.rs +++ b/src/recovery.rs @@ -14,7 +14,7 @@ use crate::packets::{HEADER_SIZE, MAGIC}; const TYPE_RECOVERY: &[u8; 16] = b"PAR 2.0\x00RecvSlic"; /// A recovery block extracted from a RecoverySlice packet. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct RecoveryBlock { /// The exponent (recovery block index) used in the Vandermonde matrix. pub exponent: u32, diff --git a/src/repair.rs b/src/repair.rs index 05e7f9a..a449c50 100644 --- a/src/repair.rs +++ b/src/repair.rs @@ -10,9 +10,44 @@ //! 5. Compute adjusted recovery: subtract intact-block contributions from recovery data //! 6. Apply the D×D inverse to produce repaired blocks //! 7. Write repaired blocks back to files +//! +//! Steps 3–6 are the [`Repair`] builder, which works on slices in memory and +//! does no I/O. The file-based functions ([`repair`], [`repair_from_verify`], +//! [`repair_from_verify_no_reverify`]) read intact slices from disk, feed them +//! to a [`Repair`], and write the solved slices back. +//! +//! # In-memory repair +//! +//! ``` +//! use rust_par2::recovery::RecoveryBlock; +//! use rust_par2::repair::Repair; +//! # fn demo(recovery: Vec, present: Vec<(usize, Vec)>) -> Result<(), rust_par2::RepairError> { +//! // Global slice indices run across files in the Main packet's file order; +//! // `BlockMap` maps them to (file, offset, length). +//! let (slice_size, total_slices) = (768_000, 1_000); +//! +//! let mut r = Repair::new_owning(recovery, slice_size, total_slices)?; +//! for (index, bytes) in &present { +//! // Check each slice against its IFSC checksum first: see `slice::SliceHasher`. +//! r.feed(*index, bytes)?; +//! } +//! let missing = [17, 18, 404]; +//! let rebuilt = r.solve(&missing)?; // index -> slice_size bytes (zero-padded) +//! # let _ = rebuilt; Ok(()) } +//! ``` +//! +//! # Memory model +//! +//! A [`Repair`] holds one accumulator per recovery row: peak memory is +//! `rows × slice_size`, plus `D × slice_size` for the solved output, plus +//! whatever the caller buffers before feeding. Feeding allocates nothing for +//! [`Repair::feed`]; [`Repair::feed_batch`] allocates one small coefficient +//! vector per row per call. To bound memory, load or keep only as many recovery +//! rows as there are missing slices ([`Repair::truncate_rows`] drops the rest at +//! any time — each row is an independent equation). -use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; use std::fmt; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::Path; @@ -24,7 +59,7 @@ use crate::gf; use crate::gf_simd; use crate::matrix::{GfMatrix, par2_input_constants}; use crate::recovery::{RecoveryBlock, load_recovery_blocks}; -use crate::types::{Par2FileSet, VerifyResult}; +use crate::types::{Id16, Par2FileSet, VerifyResult}; use crate::verify; /// Result of a repair operation. @@ -67,8 +102,310 @@ pub enum RepairError { NoDamage, #[error("Verification after repair failed: {0}")] VerifyFailed(String), + #[error("Invalid slice size {0}: must be a non-zero multiple of 4")] + InvalidSliceSize(usize), + #[error("Recovery block {exponent} is {len} bytes, expected the slice size {slice_size}")] + InvalidRecoveryBlock { + exponent: u32, + len: usize, + slice_size: usize, + }, + #[error("Slice index {index} out of range: the set has {total} input slices")] + SliceOutOfRange { index: usize, total: usize }, + #[error("Slice {index} is {len} bytes, longer than the slice size {slice_size}")] + SliceTooLong { + index: usize, + len: usize, + slice_size: usize, + }, + #[error("Slice {index} was already fed (feeding twice cancels it out)")] + DuplicateSlice { index: usize }, + #[error("{unaccounted} input slices were neither fed nor listed as missing")] + IncompleteFeed { unaccounted: usize }, +} + +// --------------------------------------------------------------------------- +// In-memory builder +// --------------------------------------------------------------------------- + +/// Incremental, I/O-free PAR2 repair over slices held in memory. +/// +/// Load recovery rows, [`feed`](Self::feed) every present input slice once, +/// then [`solve`](Self::solve) for the missing ones. Each feed XOR-accumulates +/// `c_i^exp_e × D_i` into every recovery row `e`, so the slices can arrive in +/// any order and need not be kept after they are fed. +/// +/// A fed slice cannot be removed. Feeding a corrupt slice makes every solved +/// slice wrong, so verify slices first (see +/// [`SliceHasher`](crate::slice::SliceHasher) and +/// [`Par2File::slice_matches`](crate::Par2File::slice_matches)). +/// +/// Slice indices are global: they run across the set's files in the Main +/// packet's file order. [`BlockMap`] converts between global indices and file +/// positions. +#[derive(Debug, Clone)] +pub struct Repair { + slice_size: usize, + /// PAR2 input constant for every global slice index. + constants: Vec, + /// Recovery exponent of each row, ascending. + exponents: Vec, + /// Accumulators, one per row: `R_e ⊕ Σ_fed c_i^exp_e × D_i`. + rows: Vec>, + fed: Vec, + fed_count: usize, } +impl Repair { + /// Build from borrowed recovery blocks (their data is copied). + pub fn new( + recovery: &[RecoveryBlock], + slice_size: usize, + total_input_slices: usize, + ) -> Result { + Self::new_owning(recovery.to_vec(), slice_size, total_input_slices) + } + + /// Build from owned recovery blocks, reusing their buffers as the row + /// accumulators. + /// + /// Rows are sorted by exponent and duplicate exponents dropped. Every block + /// must be exactly `slice_size` bytes. + pub fn new_owning( + mut recovery: Vec, + slice_size: usize, + total_input_slices: usize, + ) -> Result { + if slice_size == 0 || slice_size % 4 != 0 { + return Err(RepairError::InvalidSliceSize(slice_size)); + } + if let Some(bad) = recovery.iter().find(|b| b.data.len() != slice_size) { + return Err(RepairError::InvalidRecoveryBlock { + exponent: bad.exponent, + len: bad.data.len(), + slice_size, + }); + } + recovery.sort_by_key(|b| b.exponent); + recovery.dedup_by_key(|b| b.exponent); + + let (exponents, rows) = recovery.into_iter().map(|b| (b.exponent, b.data)).unzip(); + Ok(Self { + slice_size, + constants: par2_input_constants(total_input_slices), + exponents, + rows, + fed: vec![false; total_input_slices], + fed_count: 0, + }) + } + + /// Number of recovery rows loaded: the most missing slices `solve` can + /// rebuild. + pub fn rows(&self) -> usize { + self.rows.len() + } + + /// Recovery exponents of the loaded rows, ascending. + pub fn exponents(&self) -> &[u32] { + &self.exponents + } + + /// Slice size in bytes. + pub fn slice_size(&self) -> usize { + self.slice_size + } + + /// Total number of input slices in the set. + pub fn total_slices(&self) -> usize { + self.fed.len() + } + + /// Number of slices fed so far. + pub fn fed_count(&self) -> usize { + self.fed_count + } + + /// Whether slice `index` has been fed. + pub fn is_fed(&self, index: usize) -> bool { + self.fed.get(index).copied().unwrap_or(false) + } + + /// Keep only the first `n` rows (lowest exponents), freeing the rest. + /// + /// Safe at any point: each row is an independent equation. `solve` only + /// uses as many rows as there are missing slices, so once that number is + /// known the others are dead weight. + pub fn truncate_rows(&mut self, n: usize) { + self.rows.truncate(n); + self.exponents.truncate(n); + } + + /// Feed one present input slice. + /// + /// `data` is the slice as stored: `slice_size` bytes, or fewer for the last + /// slice of a file, which is treated as zero-padded. + pub fn feed(&mut self, index: usize, data: &[u8]) -> Result<(), RepairError> { + self.check_feed(index, data)?; + let c = self.constants[index]; + self.rows + .par_iter_mut() + .zip(self.exponents.par_iter()) + .for_each(|(row, &exp)| mul_add_prefix(row, data, gf::pow(c, exp))); + self.fed[index] = true; + self.fed_count += 1; + Ok(()) + } + + /// Feed several present slices at once. + /// + /// Each row is updated against the whole batch while it is hot in cache, + /// using the pair-batched SIMD path, so batches of a few dozen slices are + /// much faster than the same slices fed one by one. The batch is validated + /// before anything is accumulated: on error, nothing from it was fed. + pub fn feed_batch(&mut self, slices: &[(usize, &[u8])]) -> Result<(), RepairError> { + for (k, &(index, data)) in slices.iter().enumerate() { + if let Err(e) = self.check_feed(index, data) { + for &(earlier, _) in &slices[..k] { + self.fed[earlier] = false; + } + return Err(e); + } + // Marking as we go also rejects duplicates within the batch. + self.fed[index] = true; + } + + let slice_size = self.slice_size; + let (full, short): (Vec<_>, Vec<_>) = slices + .iter() + .copied() + .partition(|(_, data)| data.len() == slice_size); + let full_data: Vec<&[u8]> = full.iter().map(|&(_, d)| d).collect(); + + let constants = &self.constants; + self.rows + .par_iter_mut() + .zip(self.exponents.par_iter()) + .for_each(|(row, &exp)| { + let coeffs: Vec = full + .iter() + .map(|&(i, _)| gf::pow(constants[i], exp)) + .collect(); + gf_simd::mul_add_multi(row, &full_data, &coeffs); + for &(i, data) in &short { + mul_add_prefix(row, data, gf::pow(constants[i], exp)); + } + }); + + self.fed_count += slices.len(); + Ok(()) + } + + /// Rebuild the `missing` slices. + /// + /// Every slice must be either fed or listed in `missing`. Returns each + /// missing index mapped to its full `slice_size` bytes; the last slice of a + /// file comes back zero-padded, and the caller writes only its real length + /// (see [`BlockMap::global_to_file`]). + /// + /// Uses the first `missing.len()` rows. Errors with + /// [`RepairError::InsufficientRecovery`] if there are fewer rows than + /// missing slices, and [`RepairError::SingularMatrix`] if the chosen rows + /// are linearly dependent. + pub fn solve(&self, missing: &[usize]) -> Result>, RepairError> { + let mut missing = missing.to_vec(); + missing.sort_unstable(); + missing.dedup(); + let outputs = self.reconstruct(&missing)?; + Ok(missing.into_iter().zip(outputs).collect()) + } + + fn check_feed(&self, index: usize, data: &[u8]) -> Result<(), RepairError> { + let total = self.fed.len(); + if index >= total { + return Err(RepairError::SliceOutOfRange { index, total }); + } + if data.len() > self.slice_size { + return Err(RepairError::SliceTooLong { + index, + len: data.len(), + slice_size: self.slice_size, + }); + } + if self.fed[index] { + return Err(RepairError::DuplicateSlice { index }); + } + Ok(()) + } + + /// Solve for sorted, deduplicated `missing`; outputs are in the same order. + fn reconstruct(&self, missing: &[usize]) -> Result>, RepairError> { + let total = self.fed.len(); + for &index in missing { + if index >= total { + return Err(RepairError::SliceOutOfRange { index, total }); + } + if self.fed[index] { + return Err(RepairError::DuplicateSlice { index }); + } + } + let unaccounted = total - self.fed_count - missing.len(); + if unaccounted != 0 { + return Err(RepairError::IncompleteFeed { unaccounted }); + } + + let d = missing.len(); + if d == 0 { + return Ok(Vec::new()); + } + if d > self.rows.len() { + return Err(RepairError::InsufficientRecovery { + needed: d as u32, + available: self.rows.len() as u32, + }); + } + + // Rearranging the encoding equation R_e = Σ_i c_i^exp_e × D_i for the + // missing slices only: + // R_e ⊕ Σ_{fed i} c_i^exp_e × D_i = Σ_{missing j} c_j^exp_e × D_j + // The left side is what the rows hold. Invert V[e][j] = c_j^exp_e. + let mut vandermonde = GfMatrix::zeros(d, d); + for (e, &exp) in self.exponents[..d].iter().enumerate() { + for (j, &idx) in missing.iter().enumerate() { + vandermonde.set(e, j, gf::pow(self.constants[idx], exp)); + } + } + let inverse = vandermonde.invert().ok_or(RepairError::SingularMatrix)?; + debug!(d, "D×D decode matrix inverted"); + + // repaired[j] = Σ_e V⁻¹[j][e] × row[e] + let row_refs: Vec<&[u8]> = self.rows[..d].iter().map(Vec::as_slice).collect(); + let mut outputs: Vec> = (0..d).map(|_| vec![0u8; self.slice_size]).collect(); + outputs.par_iter_mut().enumerate().for_each(|(j, dst)| { + let coeffs: Vec = (0..d).map(|e| inverse.get(j, e)).collect(); + gf_simd::mul_add_multi(dst, &row_refs, &coeffs); + }); + Ok(outputs) + } +} + +/// `dst ^= c × src` over `src.len()` bytes, `src` treated as zero-padded to +/// `dst.len()`. Padding contributes nothing, so only the prefix is touched; an +/// odd trailing byte is the low half of a GF(2^16) word whose high byte is 0. +fn mul_add_prefix(dst: &mut [u8], src: &[u8], c: u16) { + let even = src.len() & !1; + gf_simd::mul_add_buffer(&mut dst[..even], &src[..even], c); + if even < src.len() { + let p = gf::mul(c, src[even] as u16); + dst[even] ^= p as u8; + dst[even + 1] ^= (p >> 8) as u8; + } +} + +// --------------------------------------------------------------------------- +// File-based repair +// --------------------------------------------------------------------------- + /// Repair damaged/missing files in a PAR2 set. /// /// This is a blocking operation. For async contexts, wrap in `spawn_blocking`. @@ -139,76 +476,36 @@ fn repair_from_verify_inner( // Step 1: Load recovery blocks from all volume files let recovery_blocks = load_recovery_blocks(dir, &file_set.recovery_set_id, file_set.slice_size); - if (recovery_blocks.len() as u32) < blocks_needed { + // Step 2: Map files to global block indices + let block_map = build_block_map(file_set); + let total_input_blocks = block_map.total_blocks as usize; + let slice_size = file_set.slice_size as usize; + + let mut fold = Repair::new_owning(recovery_blocks, slice_size, total_input_blocks)?; + if (fold.rows() as u32) < blocks_needed { return Err(RepairError::InsufficientRecovery { needed: blocks_needed, - available: recovery_blocks.len() as u32, + available: fold.rows() as u32, }); } - // Step 2: Map files to global block indices - let block_map = build_block_map(file_set); - let total_input_blocks = block_map.total_blocks as usize; - let damaged_indices = find_damaged_block_indices(verify_result, &block_map); - let num_damaged = damaged_indices.len(); info!( - damaged_block_count = num_damaged, + damaged_block_count = damaged_indices.len(), total_input_blocks, "Mapped damaged blocks to global indices" ); - // Select exactly D recovery blocks (one per damaged block) - let recovery_to_use: Vec<&RecoveryBlock> = recovery_blocks.iter().take(num_damaged).collect(); - let recovery_exponents: Vec = recovery_to_use.iter().map(|b| b.exponent).collect(); + // Exactly D recovery rows are needed (one per damaged block). + fold.truncate_rows(damaged_indices.len()); - // Step 3: Build and invert the D×D Vandermonde submatrix. - // - // The full encoding equation for recovery block e is: - // R_e = Σ_i (c_i^exp_e × D_i) for all input blocks i + // Step 3: Fold intact blocks into the recovery rows. // - // Rearranging for the damaged blocks only: - // R_e - Σ_{intact i} (c_i^exp_e × D_i) = Σ_{damaged j} (c_j^exp_e × D_j) - // - // The D×D matrix V[e][j] = c_{damaged_j}^{exp_e} is what we invert. - let constants = par2_input_constants(total_input_blocks); - - let mut vandermonde = GfMatrix::zeros(num_damaged, num_damaged); - for (e, &exp) in recovery_exponents.iter().enumerate() { - for (j, &dmg_idx) in damaged_indices.iter().enumerate() { - vandermonde.set(e, j, gf::pow(constants[dmg_idx], exp)); - } - } - - let inverse = vandermonde.invert().ok_or(RepairError::SingularMatrix)?; - - info!( - "D×D decode matrix inverted ({}×{})", - num_damaged, num_damaged - ); - - // Step 4: Compute adjusted recovery buffers. - // - // adjusted[e] = R_e ⊕ Σ_{intact i} (c_i^exp_e × D_i) - // - // In GF(2^16), subtraction = addition = XOR, so we XOR-accumulate the - // intact-block contributions into the recovery data. - // - // Cache-tiled approach: the D output buffers (D × slice_size) typically - // exceed L3 cache. Iterating source-major (one source at a time against - // all D outputs) thrashes cache on every source. Instead we read batches - // of B source blocks, then process each output buffer against the entire - // batch — keeping the output buffer hot in L1/L2 while applying B sources - // via mul_add_multi's pair-batched AVX2 path. - - let slice_size = file_set.slice_size as usize; - - // Build a fast lookup for damaged indices - let damaged_set: std::collections::HashSet = damaged_indices.iter().copied().collect(); - - // Initialize adjusted recovery from the actual recovery block data - let mut adjusted: Vec> = recovery_to_use.iter().map(|rb| rb.data.clone()).collect(); - - // Collect intact block indices for batch reading + // Cache-tiled: the D row buffers (D × slice_size) typically exceed L3. + // Iterating source-major thrashes cache on every source. Instead we read + // batches of B source blocks and feed_batch processes each row against the + // whole batch, keeping the row hot in L1/L2 while applying B sources via + // mul_add_multi's pair-batched AVX2 path. + let damaged_set: HashSet = damaged_indices.iter().copied().collect(); let intact_indices: Vec = (0..total_input_blocks) .filter(|i| !damaged_set.contains(i)) .collect(); @@ -217,68 +514,48 @@ fn repair_from_verify_inner( // With 768KB slices, 24 blocks = ~18MB. Leaves room for output buffer in L1/L2. const BATCH_SIZE: usize = 24; - // Read and process source blocks in cache-friendly batches. // I/O is sequential so OS readahead keeps the pipeline full. let mut file_handles: HashMap = HashMap::new(); for batch in intact_indices.chunks(BATCH_SIZE) { - // Read this batch of source blocks let batch_data: Vec> = batch .iter() .map(|&idx| read_source_block(dir, &block_map, idx, slice_size, &mut file_handles)) .collect::>>()?; - - let batch_refs: Vec<&[u8]> = batch_data.iter().map(|v| v.as_slice()).collect(); - - // For each output buffer (parallelized across cores), apply all sources - // in this batch at once. The output buffer stays in L1/L2 for the whole - // batch, and mul_add_multi uses pair-batched AVX2 to halve dst bandwidth. - adjusted - .par_iter_mut() - .enumerate() - .for_each(|(e, adj_buf)| { - let coeffs: Vec = batch - .iter() - .map(|&src_idx| gf::pow(constants[src_idx], recovery_exponents[e])) - .collect(); - gf_simd::mul_add_multi(adj_buf, &batch_refs, &coeffs); - }); + let pairs: Vec<(usize, &[u8])> = batch + .iter() + .copied() + .zip(batch_data.iter().map(Vec::as_slice)) + .collect(); + fold.feed_batch(&pairs)?; } info!("Intact-block contributions subtracted from recovery data"); - // Step 5: Apply the D×D inverse to produce repaired blocks. - // - // repaired[j] = Σ_e (V^{-1}[j][e] × adjusted[e]) - // - // This is D × D mul-adds of slice_size — small relative to the streaming phase. - - let adj_refs: Vec<&[u8]> = adjusted.iter().map(|v| v.as_slice()).collect(); - - let mut outputs: Vec> = (0..num_damaged).map(|_| vec![0u8; slice_size]).collect(); - - outputs.par_iter_mut().enumerate().for_each(|(j, dst)| { - let coeffs: Vec = (0..num_damaged).map(|e| inverse.get(j, e)).collect(); - gf_simd::mul_add_multi(dst, &adj_refs, &coeffs); - }); + // Steps 4–5: invert the D×D matrix and apply it. + let mut repaired = fold.solve(&damaged_indices)?; + drop(fold); info!("Repaired blocks reconstructed via D×D inverse"); - let repaired_blocks: Vec<(usize, Vec)> = - damaged_indices.iter().copied().zip(outputs).collect(); - // Step 6: Write repaired blocks back to files - let mut files_touched = std::collections::HashSet::new(); - - for (global_idx, data) in &repaired_blocks { - let (filename, file_offset, write_len) = block_map.global_to_file(*global_idx, slice_size); - - let file_path = dir.join(&filename); + let mut files_touched = HashSet::new(); + + for global_idx in &damaged_indices { + let data = repaired + .remove(global_idx) + .expect("solve returns every missing index"); + let loc = block_map + .global_to_file(*global_idx) + .expect("damaged index comes from the block map"); + let filename = &loc.file.filename; + + let file_path = dir.join(filename); debug!( filename, global_block = global_idx, - offset = file_offset, - len = write_len, + offset = loc.offset, + len = loc.len, "Writing repaired block" ); @@ -289,34 +566,29 @@ fn repair_from_verify_inner( .open(&file_path)?; // Ensure file is the right size (for missing files) - let expected_size = block_map - .files - .iter() - .find(|bf| bf.filename == filename) - .map(|bf| bf.file_size) - .unwrap_or(0); - let current_size = f.metadata()?.len(); - if current_size < expected_size { - f.set_len(expected_size)?; + if f.metadata()?.len() < loc.file.file_size { + f.set_len(loc.file.file_size)?; } - f.seek(SeekFrom::Start(file_offset as u64))?; - f.write_all(&data[..write_len])?; + f.seek(SeekFrom::Start(loc.offset))?; + f.write_all(&data[..loc.len])?; files_touched.insert(filename.clone()); } + let blocks_repaired = damaged_indices.len() as u32; + // Step 7: Optional re-verify if re_verify { let verification = verify::verify(file_set, dir); if verification.all_correct() { info!( - blocks = repaired_blocks.len(), + blocks = blocks_repaired, files = files_touched.len(), "Repair successful — all files verified" ); Ok(RepairResult { success: true, - blocks_repaired: repaired_blocks.len() as u32, + blocks_repaired, files_repaired: files_touched.len(), message: "All files repaired and verified".to_string(), }) @@ -325,13 +597,13 @@ fn repair_from_verify_inner( } } else { info!( - blocks = repaired_blocks.len(), + blocks = blocks_repaired, files = files_touched.len(), "Repair complete (re-verify skipped)" ); Ok(RepairResult { success: true, - blocks_repaired: repaired_blocks.len() as u32, + blocks_repaired, files_repaired: files_touched.len(), message: "All files repaired (re-verify skipped)".to_string(), }) @@ -342,21 +614,51 @@ fn repair_from_verify_inner( // Block mapping // --------------------------------------------------------------------------- -/// Maps between global block indices and per-file block positions. -struct BlockMap { - files: Vec, - total_blocks: u32, +/// Maps between global slice indices and per-file slice positions. +/// +/// Global indices run across the set's files in the Main packet's file order +/// (falling back to File ID order if the Main packet listed none); this is the +/// order the PAR2 input constants are assigned in. +#[derive(Debug, Clone)] +pub struct BlockMap { + /// Files in global order. + pub files: Vec, + /// Total number of input slices across all files. + pub total_blocks: u32, + /// Slice size in bytes. + pub slice_size: u64, } -struct BlockFile { - filename: String, - file_size: u64, - block_count: u32, - /// First global block index for this file. - start_block: u32, +/// One file's place in a [`BlockMap`]. +#[derive(Debug, Clone)] +pub struct BlockFile { + /// File ID from the FileDesc packet. + pub file_id: Id16, + /// Filename from the FileDesc packet. + pub filename: String, + /// File size in bytes. + pub file_size: u64, + /// Number of slices in this file. + pub block_count: u32, + /// First global slice index for this file. + pub start_block: u32, } -fn build_block_map(file_set: &Par2FileSet) -> BlockMap { +/// Where a global slice lives on disk. Returned by [`BlockMap::global_to_file`]. +#[derive(Debug, Clone, Copy)] +pub struct SliceLocation<'a> { + /// The file holding the slice. + pub file: &'a BlockFile, + /// Slice index within the file (0-based). + pub local_index: u32, + /// Byte offset of the slice within the file. + pub offset: u64, + /// Bytes of real data: `slice_size`, or less for the file's last slice. + pub len: usize, +} + +/// Build the global slice map for a file set. +pub fn build_block_map(file_set: &Par2FileSet) -> BlockMap { let slice_size = file_set.slice_size; let mut files = Vec::new(); let mut block_offset = 0u32; @@ -380,6 +682,7 @@ fn build_block_map(file_set: &Par2FileSet) -> BlockMap { f.size.div_ceil(slice_size) as u32 }; files.push(BlockFile { + file_id: f.file_id, filename: f.filename.clone(), file_size: f.size, block_count, @@ -391,24 +694,49 @@ fn build_block_map(file_set: &Par2FileSet) -> BlockMap { BlockMap { files, total_blocks: block_offset, + slice_size, } } impl BlockMap { - /// Convert a global block index to (filename, file_byte_offset, bytes_to_write). - fn global_to_file(&self, global_idx: usize, slice_size: usize) -> (String, usize, usize) { - let global = global_idx as u32; - for f in &self.files { - if global >= f.start_block && global < f.start_block + f.block_count { - let local_block = (global - f.start_block) as usize; - let file_offset = local_block * slice_size; - // Last block may be shorter than slice_size - let remaining = f.file_size as usize - file_offset; - let write_len = remaining.min(slice_size); - return (f.filename.clone(), file_offset, write_len); - } + /// Build the global slice map for a file set. Same as [`build_block_map`]. + pub fn new(file_set: &Par2FileSet) -> Self { + build_block_map(file_set) + } + + /// Locate a global slice index: its file, byte offset and real length. + /// `None` if the index is out of range. + pub fn global_to_file(&self, global_idx: usize) -> Option> { + let global = u32::try_from(global_idx).ok()?; + // Files are in start_block order; empty files end where they start and + // are skipped. + let pos = self + .files + .partition_point(|f| f.start_block + f.block_count <= global); + let file = self.files.get(pos)?; + if global < file.start_block { + return None; } - panic!("Global block index {global_idx} out of range"); + let local_index = global - file.start_block; + let offset = local_index as u64 * self.slice_size; + let len = (file.file_size - offset).min(self.slice_size) as usize; + Some(SliceLocation { + file, + local_index, + offset, + len, + }) + } + + /// Global index of slice `local_index` of the file named `filename`. + pub fn global_index(&self, filename: &str, local_index: u32) -> Option { + let f = self.file(filename)?; + (local_index < f.block_count).then(|| (f.start_block + local_index) as usize) + } + + /// The entry for the file named `filename`. + pub fn file(&self, filename: &str) -> Option<&BlockFile> { + self.files.iter().find(|f| f.filename == filename) } } @@ -416,11 +744,7 @@ fn find_damaged_block_indices(verify_result: &VerifyResult, block_map: &BlockMap let mut indices = Vec::new(); for damaged in &verify_result.damaged { - if let Some(bf) = block_map - .files - .iter() - .find(|f| f.filename == damaged.filename) - { + if let Some(bf) = block_map.file(&damaged.filename) { if damaged.damaged_block_indices.is_empty() { // No per-block info — assume all blocks damaged for i in 0..bf.block_count { @@ -436,11 +760,7 @@ fn find_damaged_block_indices(verify_result: &VerifyResult, block_map: &BlockMap } for missing in &verify_result.missing { - if let Some(bf) = block_map - .files - .iter() - .find(|f| f.filename == missing.filename) - { + if let Some(bf) = block_map.file(&missing.filename) { for i in 0..bf.block_count { indices.push((bf.start_block + i) as usize); } @@ -460,16 +780,18 @@ fn read_source_block( slice_size: usize, file_handles: &mut HashMap, ) -> std::io::Result> { - let (filename, file_offset, _) = block_map.global_to_file(global_idx, slice_size); + let loc = block_map + .global_to_file(global_idx) + .expect("intact index comes from the block map"); - let handle = match file_handles.entry(filename.clone()) { + let handle = match file_handles.entry(loc.file.filename.clone()) { Entry::Occupied(e) => e.into_mut(), Entry::Vacant(e) => { - let path = dir.join(&filename); + let path = dir.join(&loc.file.filename); e.insert(std::fs::File::open(&path)?) } }; - handle.seek(SeekFrom::Start(file_offset as u64))?; + handle.seek(SeekFrom::Start(loc.offset))?; let mut buf = vec![0u8; slice_size]; // zero-initialized for last-block padding let mut total = 0; diff --git a/src/slice.rs b/src/slice.rs new file mode 100644 index 0000000..d8a3c52 --- /dev/null +++ b/src/slice.rs @@ -0,0 +1,133 @@ +//! Per-slice checksums for in-flight verification. +//! +//! The IFSC packet stores an MD5 and a CRC32 for every input slice, computed +//! over the slice zero-padded to the full slice size. [`SliceHasher`] produces +//! the same pair from bytes in memory, so a caller can check a slice before +//! feeding it to a [`Repair`](crate::repair::Repair) builder: a corrupt slice +//! that is fed poisons every recovered slice and cannot be taken back out. + +use md5::{Digest, Md5}; + +use crate::types::{Par2File, SliceChecksum}; + +/// Incremental MD5 + CRC32 over one input slice, zero-padded to `slice_size`. +/// +/// ``` +/// use rust_par2::slice::SliceHasher; +/// +/// let mut h = SliceHasher::new(8); +/// h.update(b"abc"); +/// h.update(b"de"); +/// let sum = h.finish(); // checksums of b"abcde\0\0\0" +/// assert_eq!(sum, SliceHasher::checksum(b"abcde", 8)); +/// ``` +pub struct SliceHasher { + md5: Md5, + crc: crc32fast::Hasher, + slice_size: u64, + len: u64, +} + +impl SliceHasher { + /// Start hashing a slice of a set whose slice size is `slice_size`. + pub fn new(slice_size: u64) -> Self { + Self { + md5: Md5::new(), + crc: crc32fast::Hasher::new(), + slice_size, + len: 0, + } + } + + /// Feed the next bytes of the slice. + /// + /// Bytes beyond `slice_size` are hashed too, so the result will not match + /// any IFSC entry; callers should not pass more than one slice. + pub fn update(&mut self, data: &[u8]) { + self.md5.update(data); + self.crc.update(data); + self.len += data.len() as u64; + } + + /// Bytes fed so far, before padding. + pub fn len(&self) -> u64 { + self.len + } + + /// Whether no bytes have been fed yet. + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Zero-pad to `slice_size` and return the checksum pair. + pub fn finish(mut self) -> SliceChecksum { + const ZEROS: [u8; 4096] = [0; 4096]; + let mut pad = self.slice_size.saturating_sub(self.len); + while pad > 0 { + let n = pad.min(ZEROS.len() as u64) as usize; + self.md5.update(&ZEROS[..n]); + self.crc.update(&ZEROS[..n]); + pad -= n as u64; + } + SliceChecksum { + md5: self.md5.finalize().into(), + crc32: self.crc.finalize(), + } + } + + /// Checksum a whole slice in one call. + pub fn checksum(data: &[u8], slice_size: u64) -> SliceChecksum { + let mut h = Self::new(slice_size); + h.update(data); + h.finish() + } +} + +impl Par2File { + /// Whether `data` matches the IFSC checksum of this file's slice + /// `local_index` (0-based within the file). + /// + /// `data` is the slice as it appears in the file: `slice_size` bytes, or + /// fewer for the file's last slice. Returns `false` if the index is out of + /// range or the length is wrong for that slice. + pub fn slice_matches(&self, local_index: usize, data: &[u8], slice_size: u64) -> bool { + let Some(expected) = self.slices.get(local_index) else { + return false; + }; + let offset = local_index as u64 * slice_size; + let expected_len = self.size.saturating_sub(offset).min(slice_size); + if data.len() as u64 != expected_len { + return false; + } + SliceHasher::checksum(data, slice_size) == *expected + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn padding_matches_explicit_zeroes() { + let data = vec![0xABu8; 10_000]; + let mut padded = data.clone(); + padded.resize(16_384, 0); + let a = SliceHasher::checksum(&data, 16_384); + let b = SliceHasher::checksum(&padded, 16_384); + assert_eq!(a, b); + assert_eq!(a.crc32, crc32fast::hash(&padded)); + let md5: [u8; 16] = Md5::digest(&padded).into(); + assert_eq!(a.md5, md5); + } + + #[test] + fn incremental_matches_one_shot() { + let data: Vec = (0..5000u32).map(|i| (i * 7) as u8).collect(); + let mut h = SliceHasher::new(8192); + for chunk in data.chunks(333) { + h.update(chunk); + } + assert_eq!(h.len(), 5000); + assert_eq!(h.finish(), SliceHasher::checksum(&data, 8192)); + } +} diff --git a/src/types.rs b/src/types.rs index e29051f..4607b14 100644 --- a/src/types.rs +++ b/src/types.rs @@ -46,7 +46,7 @@ pub struct Par2File { } /// Checksum data for a single slice (block) of a file. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct SliceChecksum { /// MD5 hash of this slice. pub md5: Md5Hash, diff --git a/tests/fixtures/multi/alpha.bin b/tests/fixtures/multi/alpha.bin new file mode 100644 index 0000000..afc04f3 Binary files /dev/null and b/tests/fixtures/multi/alpha.bin differ diff --git a/tests/fixtures/multi/beta.bin b/tests/fixtures/multi/beta.bin new file mode 100644 index 0000000..6309fc8 Binary files /dev/null and b/tests/fixtures/multi/beta.bin differ diff --git a/tests/fixtures/multi/gamma.bin b/tests/fixtures/multi/gamma.bin new file mode 100644 index 0000000..3210771 Binary files /dev/null and b/tests/fixtures/multi/gamma.bin differ diff --git a/tests/fixtures/multi/multi.par2 b/tests/fixtures/multi/multi.par2 new file mode 100644 index 0000000..ac0f444 Binary files /dev/null and b/tests/fixtures/multi/multi.par2 differ diff --git a/tests/fixtures/multi/multi.vol0+8.par2 b/tests/fixtures/multi/multi.vol0+8.par2 new file mode 100644 index 0000000..e88a3bd Binary files /dev/null and b/tests/fixtures/multi/multi.vol0+8.par2 differ diff --git a/tests/slice_repair.rs b/tests/slice_repair.rs new file mode 100644 index 0000000..67959ea --- /dev/null +++ b/tests/slice_repair.rs @@ -0,0 +1,452 @@ +//! In-memory slice repair (`Repair` builder) tests. +//! +//! The `multi` fixture was made by par2cmdline 1.0.0: +//! `par2 c -s4096 -c8 -n1 multi.par2 alpha.bin beta.bin gamma.bin` +//! — 12 input slices of 4096 bytes (alpha and beta end in short slices) and 8 +//! recovery slices. + +use std::collections::HashMap; +use std::path::Path; + +use rust_par2::matrix::GfMatrix; +use rust_par2::recovery::{RecoveryBlock, load_recovery_blocks}; +use rust_par2::{BlockMap, Par2FileSet, Repair, RepairError, SliceHasher}; + +fn multi_dir() -> &'static Path { + Path::new("tests/fixtures/multi") +} + +/// Small deterministic PRNG (splitmix64) so failures reproduce. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + + /// `k` distinct values from `0..n`, sorted. + fn choose(&mut self, n: usize, k: usize) -> Vec { + let mut all: Vec = (0..n).collect(); + for i in 0..k { + let j = i + self.below(n - i); + all.swap(i, j); + } + let mut out = all[..k].to_vec(); + out.sort_unstable(); + out + } + + fn bytes(&mut self, len: usize) -> Vec { + (0..len).map(|_| self.next() as u8).collect() + } +} + +struct Fixture { + file_set: Par2FileSet, + map: BlockMap, + recovery: Vec, + /// Every input slice as stored in its file (last slice of a file is short). + slices: Vec>, +} + +fn load_multi() -> Fixture { + let dir = multi_dir(); + let file_set = rust_par2::parse(&dir.join("multi.par2")).unwrap(); + let map = BlockMap::new(&file_set); + let recovery = load_recovery_blocks(dir, &file_set.recovery_set_id, file_set.slice_size); + let slices = (0..map.total_blocks as usize) + .map(|g| { + let loc = map.global_to_file(g).unwrap(); + let data = std::fs::read(dir.join(&loc.file.filename)).unwrap(); + data[loc.offset as usize..loc.offset as usize + loc.len].to_vec() + }) + .collect(); + Fixture { + file_set, + map, + recovery, + slices, + } +} + +fn padded(data: &[u8], slice_size: usize) -> Vec { + let mut v = data.to_vec(); + v.resize(slice_size, 0); + v +} + +/// Feed every slice not in `missing`, alternating single feeds and batches. +fn feed_present(r: &mut Repair, slices: &[Vec], missing: &[usize], rng: &mut Rng) { + let present: Vec = (0..slices.len()).filter(|i| !missing.contains(i)).collect(); + let mut rest = present.as_slice(); + while !rest.is_empty() { + let take = 1 + rng.below(rest.len().min(5)); + let (now, later) = rest.split_at(take); + if take == 1 { + r.feed(now[0], &slices[now[0]]).unwrap(); + } else { + let batch: Vec<(usize, &[u8])> = + now.iter().map(|&i| (i, slices[i].as_slice())).collect(); + r.feed_batch(&batch).unwrap(); + } + rest = later; + } +} + +#[test] +fn fixture_layout() { + let fx = load_multi(); + assert_eq!(fx.file_set.slice_size, 4096); + assert_eq!(fx.map.total_blocks, 12); + assert_eq!(fx.recovery.len(), 8); + let short: Vec = (0..12) + .filter(|&g| fx.map.global_to_file(g).unwrap().len < 4096) + .collect(); + assert!(!short.is_empty(), "fixture should have short last slices"); + assert!(fx.map.global_to_file(12).is_none()); +} + +#[test] +fn slices_match_ifsc_checksums() { + let fx = load_multi(); + for (g, data) in fx.slices.iter().enumerate() { + let loc = fx.map.global_to_file(g).unwrap(); + let file = &fx.file_set.files[&loc.file.file_id]; + assert!(file.slice_matches(loc.local_index as usize, data, 4096)); + assert_eq!( + SliceHasher::checksum(data, 4096), + file.slices[loc.local_index as usize] + ); + let mut bad = data.clone(); + bad[0] ^= 1; + assert!(!file.slice_matches(loc.local_index as usize, &bad, 4096)); + } +} + +/// Every damage count from 1 to the recovery count, several sets each, solved +/// from par2cmdline's recovery data. +#[test] +fn par2cmdline_fixture_round_trip() { + let fx = load_multi(); + let n = fx.slices.len(); + let mut rng = Rng(153); + for d in 1..=fx.recovery.len() { + for _ in 0..6 { + let missing = rng.choose(n, d); + let mut r = Repair::new(&fx.recovery, 4096, n).unwrap(); + assert_eq!(r.rows(), 8); + feed_present(&mut r, &fx.slices, &missing, &mut rng); + let out = r.solve(&missing).unwrap(); + assert_eq!(out.len(), d); + for &g in &missing { + let loc = fx.map.global_to_file(g).unwrap(); + assert_eq!( + out[&g], + padded(&fx.slices[g], 4096), + "slice {g} with missing {missing:?}" + ); + let file = &fx.file_set.files[&loc.file.file_id]; + assert!(file.slice_matches(loc.local_index as usize, &out[&g][..loc.len], 4096)); + } + } + } +} + +#[test] +fn par2cmdline_fixture_too_many_missing() { + let fx = load_multi(); + let n = fx.slices.len(); + let missing: Vec = (0..9).collect(); + let mut r = Repair::new(&fx.recovery, 4096, n).unwrap(); + feed_present(&mut r, &fx.slices, &missing, &mut Rng(1)); + match r.solve(&missing) { + Err(RepairError::InsufficientRecovery { + needed: 9, + available: 8, + }) => {} + other => panic!("expected InsufficientRecovery, got {other:?}"), + } +} + +#[test] +fn truncated_rows_still_solve_and_bound_errors() { + let fx = load_multi(); + let n = fx.slices.len(); + let missing = vec![2, 5, 8]; + let mut r = Repair::new_owning(fx.recovery.clone(), 4096, n).unwrap(); + r.truncate_rows(3); + assert_eq!(r.rows(), 3); + feed_present(&mut r, &fx.slices, &missing, &mut Rng(2)); + let out = r.solve(&missing).unwrap(); + for g in missing { + assert_eq!(out[&g], padded(&fx.slices[g], 4096)); + } + + let mut r = Repair::new(&fx.recovery, 4096, n).unwrap(); + r.truncate_rows(2); + let missing = vec![0, 1, 2]; + feed_present(&mut r, &fx.slices, &missing, &mut Rng(3)); + assert!(matches!( + r.solve(&missing), + Err(RepairError::InsufficientRecovery { + needed: 3, + available: 2 + }) + )); +} + +/// Damage files on disk, repair them with the file-based path, and check the +/// builder gives the same bytes for the same damage. +#[test] +fn file_repair_and_builder_agree() { + let fx = load_multi(); + let tmp = tempfile::tempdir().unwrap(); + for e in std::fs::read_dir(multi_dir()).unwrap() { + let e = e.unwrap(); + std::fs::copy(e.path(), tmp.path().join(e.file_name())).unwrap(); + } + + // Corrupt one slice in alpha and the short last slice of beta, and delete + // gamma entirely. + let damage = |name: &str, local: u32| { + let g = fx + .map + .global_to_file(fx.map.global_index(name, local).unwrap()) + .unwrap(); + let path = tmp.path().join(name); + let mut data = std::fs::read(&path).unwrap(); + data[g.offset as usize] ^= 0xFF; + std::fs::write(&path, data).unwrap(); + }; + damage("alpha.bin", 2); + let beta_last = fx.map.file("beta.bin").unwrap().block_count - 1; + damage("beta.bin", beta_last); + std::fs::remove_file(tmp.path().join("gamma.bin")).unwrap(); + + let gamma = fx.map.file("gamma.bin").unwrap(); + let mut missing = vec![ + fx.map.global_index("alpha.bin", 2).unwrap(), + fx.map.global_index("beta.bin", beta_last).unwrap(), + ]; + missing.extend((0..gamma.block_count).map(|i| (gamma.start_block + i) as usize)); + + let pre = rust_par2::verify(&fx.file_set, tmp.path()); + assert_eq!(pre.blocks_needed() as usize, missing.len()); + let result = rust_par2::repair_from_verify(&fx.file_set, tmp.path(), &pre).unwrap(); + assert!(result.success); + assert_eq!(result.blocks_repaired as usize, missing.len()); + + let mut r = Repair::new(&fx.recovery, 4096, fx.slices.len()).unwrap(); + feed_present(&mut r, &fx.slices, &missing, &mut Rng(4)); + let out = r.solve(&missing).unwrap(); + + let mut on_disk: HashMap> = HashMap::new(); + for &g in &missing { + let loc = fx.map.global_to_file(g).unwrap(); + let file = on_disk + .entry(loc.file.filename.clone()) + .or_insert_with(|| std::fs::read(tmp.path().join(&loc.file.filename)).unwrap()); + let start = loc.offset as usize; + assert_eq!(&file[start..start + loc.len], &out[&g][..loc.len]); + assert!(out[&g][loc.len..].iter().all(|&b| b == 0)); + } + for name in ["alpha.bin", "beta.bin", "gamma.bin"] { + assert_eq!( + std::fs::read(tmp.path().join(name)).unwrap(), + std::fs::read(multi_dir().join(name)).unwrap() + ); + } +} + +/// Same equivalence on the original single-file `damaged` fixture. +#[test] +fn file_repair_and_builder_agree_on_damaged_fixture() { + let src = Path::new("tests/fixtures/damaged"); + let tmp = tempfile::tempdir().unwrap(); + for e in std::fs::read_dir(src).unwrap() { + let e = e.unwrap(); + std::fs::copy(e.path(), tmp.path().join(e.file_name())).unwrap(); + } + let file_set = rust_par2::parse(&tmp.path().join("testdata.bin.par2")).unwrap(); + let map = BlockMap::new(&file_set); + let slice_size = file_set.slice_size as usize; + let pre = rust_par2::verify(&file_set, tmp.path()); + let damaged = &pre.damaged[0]; + let missing: Vec = damaged + .damaged_block_indices + .iter() + .map(|&l| map.global_index(&damaged.filename, l).unwrap()) + .collect(); + + let before = std::fs::read(tmp.path().join("testdata.bin")).unwrap(); + let recovery = load_recovery_blocks(tmp.path(), &file_set.recovery_set_id, file_set.slice_size); + let mut r = Repair::new_owning(recovery, slice_size, map.total_blocks as usize).unwrap(); + for g in 0..map.total_blocks as usize { + if !missing.contains(&g) { + let loc = map.global_to_file(g).unwrap(); + r.feed( + g, + &before[loc.offset as usize..loc.offset as usize + loc.len], + ) + .unwrap(); + } + } + let out = r.solve(&missing).unwrap(); + + rust_par2::repair_from_verify(&file_set, tmp.path(), &pre).unwrap(); + let after = std::fs::read(tmp.path().join("testdata.bin")).unwrap(); + for g in missing { + let loc = map.global_to_file(g).unwrap(); + let start = loc.offset as usize; + assert_eq!(&after[start..start + loc.len], &out[&g][..loc.len]); + } +} + +/// Encode random file layouts with random recovery exponents, damage up to the +/// recovery count, and check the builder reproduces the original bytes. +#[test] +fn property_random_layouts_and_damage() { + let mut rng = Rng(0x5EED); + for case in 0..60 { + let slice_size = 4 * (1 + rng.below(40)); // 4..=160, exercises SIMD tails + let file_count = 1 + rng.below(4); + let mut slices: Vec> = Vec::new(); + for _ in 0..file_count { + let len = 1 + rng.below(slice_size * 6); + let data = rng.bytes(len); + slices.extend(data.chunks(slice_size).map(<[u8]>::to_vec)); + } + let n = slices.len(); + let k = 1 + rng.below(8); + let mut exponents: Vec = rng.choose(40, k).into_iter().map(|e| e as u32).collect(); + exponents.sort_unstable(); + + let enc = GfMatrix::par2_encoding_matrix(n, &exponents); + let padded_slices: Vec> = slices.iter().map(|s| padded(s, slice_size)).collect(); + let srcs: Vec<&[u8]> = padded_slices.iter().map(Vec::as_slice).collect(); + let recovery: Vec = exponents + .iter() + .enumerate() + .map(|(row, &exponent)| { + let coeffs: Vec = (0..n).map(|col| enc.get(n + row, col)).collect(); + let mut data = vec![0u8; slice_size]; + rust_par2::gf_simd::mul_add_multi(&mut data, &srcs, &coeffs); + RecoveryBlock { exponent, data } + }) + .collect(); + + let d = 1 + rng.below(k.min(n)); + let missing = rng.choose(n, d); + let mut r = Repair::new_owning(recovery, slice_size, n).unwrap(); + feed_present(&mut r, &slices, &missing, &mut rng); + let out = r.solve(&missing).unwrap_or_else(|e| { + panic!("case {case}: n={n} k={k} exps={exponents:?} missing={missing:?}: {e}") + }); + for &g in &missing { + assert_eq!(out[&g], padded_slices[g], "case {case} slice {g}"); + } + } +} + +/// A corrupt slice fed once poisons the result and cannot be taken back: +/// re-feeding the good bytes is refused. Callers must verify before feeding. +#[test] +fn corrupt_feed_cannot_be_undone() { + let fx = load_multi(); + let n = fx.slices.len(); + let missing = vec![3]; + let mut r = Repair::new(&fx.recovery, 4096, n).unwrap(); + + let mut corrupt = fx.slices[0].clone(); + corrupt[100] ^= 0x55; + r.feed(0, &corrupt).unwrap(); + assert!(matches!( + r.feed(0, &fx.slices[0]), + Err(RepairError::DuplicateSlice { index: 0 }) + )); + for g in 1..n { + if g != 3 { + r.feed(g, &fx.slices[g]).unwrap(); + } + } + let out = r.solve(&missing).unwrap(); + assert_ne!(out[&3], padded(&fx.slices[3], 4096)); + + // The slice hasher would have caught it. + let loc = fx.map.global_to_file(0).unwrap(); + let file = &fx.file_set.files[&loc.file.file_id]; + assert!(!file.slice_matches(0, &corrupt, 4096)); +} + +#[test] +fn input_validation() { + let fx = load_multi(); + let n = fx.slices.len(); + let mut r = Repair::new(&fx.recovery, 4096, n).unwrap(); + + assert!(matches!( + r.feed(n, &fx.slices[0]), + Err(RepairError::SliceOutOfRange { index, total }) if index == n && total == n + )); + assert!(matches!( + r.feed(0, &[0u8; 4097]), + Err(RepairError::SliceTooLong { len: 4097, .. }) + )); + + // A bad batch feeds nothing, including the valid entries before the error. + let bad_batch: Vec<(usize, &[u8])> = + vec![(0, &fx.slices[0]), (1, &fx.slices[1]), (1, &fx.slices[1])]; + assert!(matches!( + r.feed_batch(&bad_batch), + Err(RepairError::DuplicateSlice { index: 1 }) + )); + assert_eq!(r.fed_count(), 0); + assert!(!r.is_fed(0)); + + // Slices neither fed nor missing. + r.feed(0, &fx.slices[0]).unwrap(); + assert!(matches!( + r.solve(&[1]), + Err(RepairError::IncompleteFeed { unaccounted }) if unaccounted == n - 2 + )); + // A fed slice cannot also be missing. + assert!(matches!( + r.solve(&[0]), + Err(RepairError::DuplicateSlice { index: 0 }) + )); + + assert!(matches!( + Repair::new(&fx.recovery, 4098, n), + Err(RepairError::InvalidSliceSize(4098)) + )); + assert!(matches!( + Repair::new(&fx.recovery, 8192, n), + Err(RepairError::InvalidRecoveryBlock { len: 4096, .. }) + )); +} + +#[test] +fn duplicate_recovery_exponents_are_dropped() { + let fx = load_multi(); + let mut doubled = fx.recovery.clone(); + doubled.extend(fx.recovery.iter().cloned()); + let r = Repair::new_owning(doubled, 4096, fx.slices.len()).unwrap(); + assert_eq!(r.rows(), 8); + assert!(r.exponents().windows(2).all(|w| w[0] < w[1])); +} + +#[test] +fn nothing_missing_solves_to_empty() { + let fx = load_multi(); + let mut r = Repair::new(&fx.recovery, 4096, fx.slices.len()).unwrap(); + feed_present(&mut r, &fx.slices, &[], &mut Rng(5)); + assert!(r.solve(&[]).unwrap().is_empty()); +}