From 49c7ab4bf5e0b8db46cc0961599c2f5d9dbf9c2f Mon Sep 17 00:00:00 2001 From: cppbear Date: Fri, 9 Oct 2026 10:48:57 +0800 Subject: [PATCH] fix: collect target-specific zero-hit coverage baselines --- .github/workflows/ci.yml | 3 + Cargo.lock | 1 + docs/minimal-pipeline.md | 6 +- docs/palm-rust-unit-test-generation.md | 2 + scripts/check_minimal.py | 27 +- utgen/Cargo.toml | 1 + utgen/README.md | 4 + utgen/src/gene/mod.rs | 1 + utgen/src/run/coverage.rs | 62 ++-- utgen/src/run/coverage_baseline.rs | 431 +++++++++++++++++++++++ utgen/src/run/coverage_baseline_tests.rs | 226 ++++++++++++ utgen/src/run/llm_fix.rs | 1 + utgen/src/run/mod.rs | 2 + utgen/src/run/run.rs | 88 ++--- 14 files changed, 786 insertions(+), 69 deletions(-) create mode 100644 utgen/src/run/coverage_baseline.rs create mode 100644 utgen/src/run/coverage_baseline_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9607cc610..e1c31409df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,9 @@ jobs: - name: Check coverage annotation compatibility if: steps.scope.outputs.run_code == 'true' run: python3 scripts/check_coverage.py + - name: Check coverage baselines for unit and integration builds + if: steps.scope.outputs.run_code == 'true' + run: cargo test -p utgen --locked coverage_baselines_match_test_build_modes -- --ignored - name: Validate the minimal analysis and test pipeline if: steps.scope.outputs.run_code == 'true' run: python3 scripts/check_minimal.py diff --git a/Cargo.lock b/Cargo.lock index 3798606641..59d7907854 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2073,6 +2073,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "shlex", "simplelog", "strfmt", "syn", diff --git a/docs/minimal-pipeline.md b/docs/minimal-pipeline.md index 857da26814..db11634433 100644 --- a/docs/minimal-pipeline.md +++ b/docs/minimal-pipeline.md @@ -17,6 +17,7 @@ Run the related checks when changing coverage/preprocessing or task scheduling: ```sh python3 scripts/check_coverage.py +cargo test -p utgen --locked coverage_baselines_match_test_build_modes -- --ignored python3 scripts/check_tasks.py ``` @@ -41,7 +42,8 @@ The default binary directory is `target/debug`. The coverage tool must be availa | Coverage | Excludes test bodies and test-only helpers, preserves production coverage, and executes tests once when exporting XML and JSON. | | Cleanup | Restores sources after the covered success, assertion-failure, compilation-failure, and report-export-failure cases. | | Candidate timeout | Ordinary and `should_panic` infinite loops count as failures; an expected panic and a later ordinary test pass in unit and integration statistics. | -| Statistics | Oracle groups count once across multiple candidates; pass results match complete test names; an all-compilation-failure result reports unavailable coverage without exporting an empty run. | +| Statistics | Oracle groups count once across multiple candidates; pass results match complete test names; all-compilation-failure results retain a zero-hit denominator without a synthetic candidate. A passing integration candidate that does not call the focal function has zero focal coverage. | +| Coverage baselines | No-test unit/library and binary harnesses; ordinary-library integration maps; conditional compilation, generics, no-map targets, per-target reuse, separation from real profiles, and source/helper restoration after build failure. | | Integration repair | Repairs alias/trait imports, helpers, and test bodies in integration targets; keeps sibling candidates independent; checks cached repair, mode mismatch, unselected targets, and cleanup after request failure. | | Integration directories | Original and generated tests coexist after normal runs and cached reruns; original-test compilation failure restores both directories; an existing staging directory is preserved and reported. | @@ -61,6 +63,8 @@ The minimal check's temporary directory contains `logs/`, separate `original/` a Explicit `coverage(on)` is respected. Expression fragments are preserved; the checks do not provide general macro expansion or infer that arbitrary unmarked helpers are test-only. See the [coverage reference](../utgen/README.md#coverage-of-existing-tests) for supported scope. +The baseline test is normally ignored because it needs the pinned coverage and LLVM tools. The explicit command above and Linux CI run it after installing those tools. It compares ordinary-library baseline maps with a real integration run, checks that a binary's `main` and existing test bodies never run during baseline collection, and removes its temporary fixtures afterward. Per-function `coverage_available=false` values must be excluded from coverage aggregates; they do not mean zero coverage of a known denominator. + ## Task Scheduling Regression `check_tasks.py` exercises function limits N=1, N=2, and the default N=4, with more than 2N functions in each scheduling case. It checks overlapping model requests, serial compiler activity, bounded-queue backpressure, and completion after queued work resumes. Cargo's target directory is reused between scenarios. diff --git a/docs/palm-rust-unit-test-generation.md b/docs/palm-rust-unit-test-generation.md index e8f5d7f18f..82b13ed2a4 100644 --- a/docs/palm-rust-unit-test-generation.md +++ b/docs/palm-rust-unit-test-generation.md @@ -424,6 +424,8 @@ export PALM_CONFIG="$(pwd)/utgen/res/api.json" 最终统计以 `utgen/result/` 和 `utgen/fixed_result/` 中的 JSON 为准；当前实现不生成 HTML 报告。`coverage_available=false` 表示没有取得被测函数的覆盖映射，覆盖数值仅为占位，不应参与汇总，也不能直接解释为条件编译排除了函数。旧结果缺少该字段时，其可用性未确认。`tests_*` 按候选计数，`oracles_*` 按原有 TestInfo 分组计数；一组中有候选通过，该组即计为通过。运行数保留包含 ignored 条目的既有口径。 +候选覆盖率缺少被测函数映射时，统计会补采集零命中基线。单元测试使用 `--lib` 或 `--bin` 的测试编译产物，过滤全部测试，不再插入空测试；集成测试从普通库产物取得映射，因此能够保留 `#[cfg(not(test))]` 下的函数和分支。基线不增加测试或 oracle 计数，不生成候选覆盖记录，也不混入真实命中数据；同一轮统计按目标复用基线，编译或工具错误仍然报错。函数级集成测试需要库目标；同名库与二进制目标共享模块、无法唯一确定函数归属时，会在模型请求前拒绝执行，完整支持需要进一步完善分析索引中的目标身份。 + ### 8.7 LLM 修复流程 ![编译错误修复流程](assets/palm-fix-flow.png) diff --git a/scripts/check_minimal.py b/scripts/check_minimal.py index 23455cf698..71b4cf0896 100644 --- a/scripts/check_minimal.py +++ b/scripts/check_minimal.py @@ -319,6 +319,9 @@ def do_POST(self): for _, branches in classify['codes_branches_covered'] for branch in branches), classify double = json.loads((target / f"utgen/result/{names['palm_fixture::nested::double']}.json").read_text()) assert double['tests_compiled'] == 0 and double['tests'] == 1, double + assert double['coverage_available'] and double['lines'] == 3, double + assert double['lines_covered'] == double['branches_covered'] == 0, double + assert double['codes_lines_covered'] == double['codes_branches_covered'] == [], double assert not list((target / 'src').rglob('*.bak')) # Replay cached candidates on one function: ordinary and should_panic @@ -371,7 +374,7 @@ def do_POST(self): assert snapshot(target) == prepared # Multiple candidates belong to one oracle; all failed compilation is - # still a valid result and must not attempt to export empty coverage. + # still a valid result with a zero-hit ordinary-library denominator. data['integration'] = True answer['common'] = [] answer['chain_tests'] = [dict( @@ -396,9 +399,25 @@ def do_POST(self): assert result['tests'] == result['oracles'] == 1, result assert result['tests_compiled'] == result['tests_run'] == result['tests_passed'] == 0, result assert result['oracles_compiled'] == result['oracles_run'] == result['oracles_passed'] == 0, result - assert not result['coverage_available'], result - assert result['lines'] == result['branches'] == 0, result - assert result['codes_lines_covered'] == [], result + assert result['coverage_available'], result + assert result['lines'] == 5 and result['branches'] == 2, result + assert result['lines_covered'] == result['branches_covered'] == 0, result + assert result['codes_lines_covered'] == result['codes_branches_covered'] == [], result + test['codes'] = [['{', 'assert_eq!(palm_fixture::nested::double(2), 4);', '}']] + test['can_compile'] = [{'Ok': None}] + candidate.write_text(json.dumps(data)) + run('integration-no-focal-call', command) + result = json.loads((target / f'utgen/result/{names[selected]}.json').read_text()) + assert result['tests_run'] == result['tests_passed'] == 1, result + assert result['coverage_available'] and result['lines'] == 5 and result['branches'] == 2, result + assert result['lines_covered'] == result['branches_covered'] == 0, result + test['codes'] = [['{', 'assert_eq!(2 + 2, 4);', '}']] + candidate.write_text(json.dumps(data)) + run('integration-no-library-call', command) + result = json.loads((target / f'utgen/result/{names[selected]}.json').read_text()) + assert result['tests_run'] == result['tests_passed'] == 1, result + assert result['coverage_available'] and result['lines'] == 5 and result['branches'] == 2, result + assert result['lines_covered'] == result['branches_covered'] == 0, result shutil.rmtree(target / 'tests') assert snapshot(target) == prepared diff --git a/utgen/Cargo.toml b/utgen/Cargo.toml index e794c2e72d..dbf36c58c6 100644 --- a/utgen/Cargo.toml +++ b/utgen/Cargo.toml @@ -21,6 +21,7 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" simplelog = "0.12" +shlex = "1.3" strfmt = "0.2" syn = { version = "2.0", features = ["full", "visit"] } time = "0.3" diff --git a/utgen/README.md b/utgen/README.md index f7c01a7ad2..89ebea3864 100644 --- a/utgen/README.md +++ b/utgen/README.md @@ -218,6 +218,10 @@ Paths under `utgen/` below are relative to `--project-dir`: Per-function results include `coverage_available`. When it is false, the numeric coverage fields are placeholders and must not enter coverage aggregates; report the number of unavailable functions separately. Missing mappings are not assumed to mean `cfg(test)` exclusion. Old results without this field have unconfirmed availability. Candidate/compilation statistics remain valid when no candidate compiles. +When candidate coverage has no focal-function mapping, statistics collect a zero-hit baseline for the selected Cargo target. Unit mode builds the `--lib` or `--bin` test harness with every test filtered out; it does not insert an empty test. Integration mode obtains mappings from the ordinary library artifact, using a temporary integration harness without executing tests. Thus `cfg(not(test))` code can contribute to the integration denominator while being absent in unit mode. Compilation and tool failures remain errors. Baselines add no candidate/oracle counts or `codes_*_covered` entries; their profiles stay separate from real test hits. Parsed baselines are reused only within the current statistics invocation. + +Function-level integration mode requires a library target. Target selection currently uses the analyzed crate name and, when needed, the source entry file. Ambiguous shared modules in same-name library/binary targets are rejected before model requests. Full mixed-target analysis requires retaining target identity in the analysis index; use a standalone library or binary working copy for those cases. + `tests_*` count candidates; `oracles_*` count the existing `TestInfo` groups. A group counts once if any of its candidates compiles, is registered to run, or passes, respectively. Run counts retain the existing libtest convention, including ignored tests. Integration pass outcomes match complete test names. `coverage.xml`, `coverage.json`, and `error_output.json` are intermediate files in the work directory and may be deleted after generation/repair parses them. The standalone `coverage` command retains both reports. The current implementation does not produce an HTML report. diff --git a/utgen/src/gene/mod.rs b/utgen/src/gene/mod.rs index d6ba319fd0..08ca26e5c2 100644 --- a/utgen/src/gene/mod.rs +++ b/utgen/src/gene/mod.rs @@ -73,6 +73,7 @@ pub async fn gen_tests_project( )); } validate_analysis(project_dir, work_dir)?; + crate::run::validate_targets(project_dir, work_dir, functions, integration)?; let saved_dir = project_dir.join("utgen/generation/pre_fix"); if saved_dir.is_dir() { for entry in fs::read_dir(saved_dir)? { diff --git a/utgen/src/run/coverage.rs b/utgen/src/run/coverage.rs index 2d988ed4c9..6020a638c5 100644 --- a/utgen/src/run/coverage.rs +++ b/utgen/src/run/coverage.rs @@ -184,20 +184,14 @@ pub fn collect_coverage(work_dir: &Path, json: bool) -> io::Result { collect_coverage_for_tests(work_dir, json, &[]) } -pub(crate) fn collect_coverage_for_tests( - work_dir: &Path, - json: bool, - targets: &[String], -) -> io::Result { - let canonical_dir = work_dir.canonicalize()?; - let work_dir = canonical_dir.as_path(); +pub(crate) fn project_metadata(work_dir: &Path) -> io::Result { let metadata = MetadataCommand::new() .manifest_path(work_dir.join("Cargo.toml")) .current_dir(work_dir) .no_deps() .exec() .map_err(io::Error::other)?; - let package = metadata.root_package().ok_or_else(|| { + metadata.root_package().ok_or_else(|| { io::Error::other("coverage requires a standalone crate with Cargo.toml and src/") })?; if metadata.workspace_members.len() != 1 || !work_dir.join("src").is_dir() { @@ -205,6 +199,15 @@ pub(crate) fn collect_coverage_for_tests( "coverage currently supports one standalone crate", )); } + Ok(metadata) +} + +pub(crate) fn with_test_exclusions( + work_dir: &Path, + metadata: &cargo_metadata::Metadata, + action: impl FnOnce() -> io::Result, +) -> io::Result { + let package = metadata.root_package().unwrap(); let roots: BTreeSet = package .targets .iter() @@ -237,6 +240,33 @@ pub(crate) fn collect_coverage_for_tests( for (path, _, annotated) in &changes { fs::write(path, annotated)?; } + action() + })(); + // Restore even when compilation, test launch, or report export failed. + // Try every changed file before reporting a restoration error. + let mut restore_error = None; + for (path, original, _) in changes { + if let Err(err) = fs::write(&path, original) { + restore_error.get_or_insert_with(|| { + io::Error::other(format!("failed to restore {}: {err}", path.display())) + }); + } + } + if let Some(err) = restore_error { + return Err(err); + } + result +} + +pub(crate) fn collect_coverage_for_tests( + work_dir: &Path, + json: bool, + targets: &[String], +) -> io::Result { + let canonical = work_dir.canonicalize()?; + let work_dir = canonical.as_path(); + let metadata = project_metadata(work_dir)?; + with_test_exclusions(work_dir, &metadata, || { // The first command uses llvm-cov's default cleanup. --no-report would // imply --no-clean and mix profiles from different generated candidates. let mut args = Vec::new(); @@ -262,21 +292,7 @@ pub(crate) fn collect_coverage_for_tests( )?; } Ok(output) - })(); - // Restore even when compilation, test launch, or report export failed. - // Try every changed file before reporting a restoration error. - let mut restore_error = None; - for (path, original, _) in changes { - if let Err(err) = fs::write(&path, original) { - restore_error.get_or_insert_with(|| { - io::Error::other(format!("failed to restore {}: {err}", path.display())) - }); - } - } - if let Some(err) = restore_error { - return Err(err); - } - result + }) } #[cfg(test)] diff --git a/utgen/src/run/coverage_baseline.rs b/utgen/src/run/coverage_baseline.rs new file mode 100644 index 0000000000..4239585b63 --- /dev/null +++ b/utgen/src/run/coverage_baseline.rs @@ -0,0 +1,431 @@ +use super::{ + coverage::{project_metadata, with_test_exclusions}, + coverage_json::CoverageJson, +}; +use crate::{FunctionSelection, types::BrData, utils::TemporaryFile}; +use cargo_metadata::{Message, Metadata, Target, TargetKind}; +use std::{ + collections::{BTreeSet, HashMap}, + fs, io, + path::{Path, PathBuf}, + process::{Command, Output}, + time::{SystemTime, UNIX_EPOCH}, +}; + +const BASELINE_TARGET: &str = "palm_coverage_baseline"; + +/// Reject unsupported target ownership before model requests or source edits. +pub(crate) fn validate_targets( + project_dir: &Path, + work_dir: &Path, + functions: &FunctionSelection, + integration: bool, +) -> io::Result<()> { + let metadata = project_metadata(work_dir)?; + if integration + && !metadata + .root_package() + .unwrap() + .targets + .iter() + .any(|target| target.is_kind(TargetKind::Lib)) + { + return Err(io::Error::other( + "Function-level integration tests require a library target", + )); + } + for entry in fs::read_dir(work_dir.join("brinfo/brdata"))? { + let path = entry?.path(); + if !path.is_file() { + continue; + } + let data: BrData = serde_json::from_slice(&fs::read(path)?)?; + if functions.contains(&data.name) && (!integration || data.visible) { + select_target( + &metadata, + &data.name, + &project_dir.join(data.loc.get_file()), + integration, + )?; + } + } + Ok(()) +} + +#[derive(Debug, Default, PartialEq)] +pub(crate) struct FunctionMapping { + pub lines: Vec, + pub branches: Vec<(i32, i32, i32, i32)>, +} + +struct Baseline { + lines: HashMap>, + branches: CoverageJson, +} + +impl Baseline { + fn function(&self, file: &Path, begin: i32, end: i32) -> FunctionMapping { + let lines = self + .lines + .get(file) + .into_iter() + .flatten() + .filter(|&&line| begin <= line && line <= end) + .copied() + .collect(); + let mut branches = BTreeSet::new(); + for data in &self.branches.data { + for entry in &data.files { + if Path::new(&entry.file_name) != file { + continue; + } + for branch in &entry.coverage_branches { + if begin <= branch.start_line as i32 && branch.end_line as i32 <= end { + branches.insert(( + branch.start_line as i32, + branch.start_column as i32, + branch.end_line as i32, + branch.end_column as i32, + )); + } + } + } + } + FunctionMapping { + lines, + branches: branches.into_iter().collect(), + } + } +} + +/// Parsed maps live only for this statistics invocation, separately for each target. +pub(crate) struct BaselineCache { + metadata: Metadata, + integration: bool, + reports: HashMap, +} + +impl BaselineCache { + pub fn new(work_dir: &Path, integration: bool) -> io::Result { + Ok(Self { + metadata: project_metadata(work_dir)?, + integration, + reports: HashMap::new(), + }) + } + + pub fn mapping( + &mut self, + work_dir: &Path, + name: &str, + file: &Path, + begin: i32, + end: i32, + ) -> io::Result { + let target = select_target(&self.metadata, name, file, self.integration)?.clone(); + let key = format!("{:?}:{}", target.kind, target.name); + if !self.reports.contains_key(&key) { + log::info!( + "Collecting {} coverage baseline for {}", + if self.integration { + "integration" + } else { + "unit" + }, + target.name + ); + let report = collect(work_dir, &self.metadata, &target, self.integration)?; + self.reports.insert(key.clone(), report); + } + Ok(self.reports[&key].function(file, begin, end)) + } +} + +fn select_target<'a>( + metadata: &'a Metadata, + name: &str, + file: &Path, + integration: bool, +) -> io::Result<&'a Target> { + let crate_name = name.split("::").next().unwrap(); + let candidates: Vec<_> = metadata + .root_package() + .unwrap() + .targets + .iter() + .filter(|t| t.is_kind(TargetKind::Lib) || t.is_kind(TargetKind::Bin)) + .filter(|t| t.name.replace('-', "_") == crate_name) + .collect(); + let target = if candidates.len() == 1 { + Some(candidates[0]) + } else { + let roots: Vec<_> = candidates + .into_iter() + .filter(|t| t.src_path.as_std_path() == file) + .collect(); + if roots.len() == 1 { + Some(roots[0]) + } else { + None + } + }; + let target = target.ok_or_else(|| io::Error::other(format!( + "Cannot uniquely select a Cargo target for {name}; use a standalone library or binary working copy" + )))?; + if integration && !target.is_kind(TargetKind::Lib) { + return Err(io::Error::other( + "Function-level integration tests require a library target", + )); + } + Ok(target) +} + +struct Scratch(PathBuf); +impl Scratch { + fn new() -> io::Result { + let path = std::env::temp_dir().join(format!( + "palm-coverage-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&path)?; + Ok(Self(path)) + } +} +impl Drop for Scratch { + fn drop(&mut self) { + if let Err(error) = fs::remove_dir_all(&self.0) { + log::warn!( + "Cannot remove coverage scratch directory {}: {error}", + self.0.display() + ); + } + } +} + +fn checked(command: &mut Command) -> io::Result { + check_output(command.output()?) +} + +fn check_output(output: Output) -> io::Result { + if !output.status.success() { + return Err(io::Error::other(format!( + "Coverage baseline command failed ({}):\n{}\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ))); + } + Ok(output) +} + +fn llvm_tools(work_dir: &Path) -> io::Result { + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let sysroot = checked( + Command::new(&rustc) + .args(["--print", "sysroot"]) + .current_dir(work_dir), + )?; + let version = checked(Command::new(&rustc).arg("-vV").current_dir(work_dir))?; + let version = String::from_utf8_lossy(&version.stdout); + let host = version + .lines() + .find_map(|line| line.strip_prefix("host: ")) + .ok_or_else(|| io::Error::other("rustc did not report its host triple"))?; + Ok( + PathBuf::from(String::from_utf8_lossy(&sysroot.stdout).trim()) + .join("lib/rustlib") + .join(host) + .join("bin"), + ) +} + +fn collect( + work_dir: &Path, + metadata: &Metadata, + target: &Target, + integration: bool, +) -> io::Result { + let scratch = Scratch::new()?; + let tools = llvm_tools(work_dir)?; + let target_dir = std::env::var_os("CARGO_LLVM_COV_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + metadata + .target_directory + .as_std_path() + .join("llvm-cov-target") + }); + let output = checked( + Command::new("cargo") + .args(["llvm-cov", "show-env", "--branch"]) + .env("CARGO_LLVM_COV_TARGET_DIR", &target_dir) + .current_dir(work_dir), + )?; + // Decode assignments as data, never execute show-env output in a shell. + let assignments = shlex::split(&String::from_utf8_lossy(&output.stdout)) + .ok_or_else(|| io::Error::other("Cannot parse cargo llvm-cov show-env output"))?; + let mut command = Command::new("cargo"); + for assignment in &assignments { + let (key, value) = assignment + .split_once('=') + .ok_or_else(|| io::Error::other("Invalid coverage environment assignment"))?; + command.env(key, value); + } + command + .env("CARGO_TARGET_DIR", &target_dir) + .env( + "LLVM_PROFILE_FILE", + scratch.0.join("baseline-%p-%m.profraw"), + ) + .current_dir(work_dir) + .arg("test"); + let temporary = if integration { + let path = work_dir.join(format!("tests/{BASELINE_TARGET}.rs")); + fs::create_dir_all(path.parent().unwrap())?; + let guard = TemporaryFile::new(&path)?; + fs::write( + &path, + format!( + "#![feature(coverage_attribute)]\nextern crate {};\n#[test]\n#[coverage(off)]\nfn baseline() {{}}\n", + target.name.replace('-', "_") + ), + )?; + command.args(["--test", BASELINE_TARGET]); + Some(guard) + } else { + if target.is_kind(TargetKind::Lib) { + command.arg("--lib"); + } else { + command.args(["--bin", &target.name]); + } + None + }; + command.args(["--message-format=json", "--", "--skip", ""]); + let output = with_test_exclusions(work_dir, metadata, || checked(&mut command))?; + if let Some(temporary) = temporary { + temporary.finish()?; + } + let package_id = &metadata.root_package().unwrap().id; + let mut objects = Vec::new(); + for message in Message::parse_stream(output.stdout.as_slice()) { + let Message::CompilerArtifact(artifact) = message.map_err(io::Error::other)? else { + continue; + }; + if &artifact.package_id != package_id + || artifact.target.name != target.name + || artifact.target.kind != target.kind + { + continue; + } + if integration && !artifact.profile.test { + for filename in artifact + .filenames + .iter() + .filter(|file| file.extension() == Some("rlib")) + { + let directory = scratch.0.join("objects"); + fs::create_dir(&directory)?; + checked( + Command::new(tools.join("llvm-ar")) + .arg("xo") + .arg(filename) + .current_dir(&directory), + )?; + for entry in fs::read_dir(directory)? { + let path = entry?.path(); + if path.extension().is_some_and(|ext| ext == "o") { + objects.push(path); + } + } + } + } else if !integration && artifact.profile.test { + if let Some(executable) = artifact.executable { + objects.push(executable.into_std_path_buf()); + } + } + } + if objects.is_empty() { + return Err(io::Error::other( + "Cargo did not produce the selected coverage artifact", + )); + } + let profile = scratch.0.join("baseline.profdata"); + let mut merge = Command::new(tools.join("llvm-profdata")); + merge.args(["merge", "-sparse"]).arg("-o").arg(&profile); + for entry in fs::read_dir(&scratch.0)? { + let path = entry?.path(); + if path.extension().is_some_and(|ext| ext == "profraw") { + merge.arg(path); + } + } + checked(&mut merge)?; + let export = |format: &str| -> io::Result { + let mut command = Command::new(tools.join("llvm-cov")); + command + .arg("export") + .arg(format!("--format={format}")) + .arg("--instr-profile") + .arg(&profile); + for object in &objects { + command.arg("--object").arg(object); + } + let output = command.output()?; + let errors = String::from_utf8_lossy(&output.stderr); + // LLVM reports an error when a successfully built target has no maps + // at all (for example, every function is cfg(not(test))). Recognize + // only that diagnostic; profile, artifact and tool failures still fail. + if !output.status.success() + && errors + .lines() + .any(|line| line.ends_with(": no coverage data found")) + && errors.lines().all(|line| { + (line.starts_with("error: failed to load coverage: ") + && line.ends_with(": no coverage data found")) + || line == "error: could not load coverage information" + }) + { + return Ok(if format == "lcov" { + "" + } else { + "{\"data\":[]}" + } + .to_owned()); + } + let output = check_output(output)?; + String::from_utf8(output.stdout).map_err(io::Error::other) + }; + parse_report(&export("lcov")?, &export("text")?) +} + +fn parse_report(lcov: &str, json: &str) -> io::Result { + let mut lines: HashMap> = HashMap::new(); + let mut current_file = None; + for line in lcov.lines() { + if let Some(file) = line.strip_prefix("SF:") { + current_file = Some(PathBuf::from(file)); + } else if let Some(record) = line.strip_prefix("DA:") { + let number = record + .split(',') + .next() + .unwrap() + .parse() + .map_err(io::Error::other)?; + if let Some(file) = ¤t_file { + lines.entry(file.clone()).or_default().insert(number); + } + } else if line == "end_of_record" { + current_file = None; + } + } + let mut branches: CoverageJson = serde_json::from_str(json).map_err(io::Error::other)?; + branches.parse_coverage_branches(); + Ok(Baseline { lines, branches }) +} + +#[cfg(test)] +#[path = "coverage_baseline_tests.rs"] +mod tests; diff --git a/utgen/src/run/coverage_baseline_tests.rs b/utgen/src/run/coverage_baseline_tests.rs new file mode 100644 index 0000000000..f320874537 --- /dev/null +++ b/utgen/src/run/coverage_baseline_tests.rs @@ -0,0 +1,226 @@ +use super::super::coverage::collect_coverage_for_tests; +use super::*; + +const SOURCE: &str = r#"pub fn ordinary(value: i32) -> i32 { + if value > 0 { 1 } else { 0 } +} +pub fn conditional(value: i32) -> i32 { + #[cfg(test)] + { value + 1 } + #[cfg(not(test))] + { if value > 0 { 1 } else { 0 } } +} +#[cfg(not(test))] +pub fn production_only(value: i32) -> i32 { + if value > 0 { 1 } else { 0 } +} +pub fn generic(a: T, b: T) -> T { + if a > b { a } else { b } +} +pub struct Container(pub T); +impl Container { + pub fn get(&self) -> &T { &self.0 } +} +#[test] +fn must_not_run() { panic!("baseline executed an existing test"); } +"#; + +fn fixture(directory: &Path, source: &str, binary: bool) -> PathBuf { + fs::create_dir_all(directory.join("src")).unwrap(); + fs::write(directory.join("Cargo.toml"), "[package]\nname = \"baseline_fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n").unwrap(); + fs::write( + directory.join("rust-toolchain.toml"), + include_str!("../../../rust-toolchain.toml"), + ) + .unwrap(); + let file = directory.join(if binary { "src/main.rs" } else { "src/lib.rs" }); + fs::write(&file, source).unwrap(); + file +} + +fn range(marker: &str) -> (i32, i32) { + let start = SOURCE.find(marker).unwrap(); + let end = start + SOURCE[start..].find("\n}").unwrap(); + ( + SOURCE[..start].lines().count() as i32 + 1, + SOURCE[..end].lines().count() as i32 + 1, + ) +} + +#[test] +#[ignore = "requires the pinned cargo-llvm-cov and llvm-tools-preview"] +fn coverage_baselines_match_test_build_modes() { + let scratch = Scratch::new().unwrap(); + let work = scratch.0.canonicalize().unwrap(); + let file = fixture(&work, SOURCE, false); + fs::create_dir(work.join("tests")).unwrap(); + let helper = work.join(format!("tests/{BASELINE_TARGET}.rs")); + fs::write(&helper, "// existing file must survive\n").unwrap(); + let markers = [ + "pub fn ordinary", + "pub fn conditional", + "pub fn production_only", + "pub fn generic", + "impl Container", + ]; + let mut unit = BaselineCache::new(&work, false).unwrap(); + let mut integration = BaselineCache::new(&work, true).unwrap(); + let mut unit_maps = Vec::new(); + let mut integration_maps = Vec::new(); + for marker in markers { + let (begin, end) = range(marker); + unit_maps.push( + unit.mapping(&work, "baseline_fixture::focal", &file, begin, end) + .unwrap(), + ); + integration_maps.push( + integration + .mapping(&work, "baseline_fixture::focal", &file, begin, end) + .unwrap(), + ); + } + assert_eq!(fs::read_to_string(&file).unwrap(), SOURCE); + assert_eq!( + fs::read_to_string(&helper).unwrap(), + "// existing file must survive\n" + ); + assert_eq!(unit.reports.len(), 1); + assert_eq!(integration.reports.len(), 1); + for index in [0, 3, 4] { + assert!(!unit_maps[index].lines.is_empty()); + assert_eq!(unit_maps[index], integration_maps[index]); + } + assert_eq!(unit_maps[1].branches.len(), 0); + assert_eq!(integration_maps[1].branches.len(), 1); + assert_eq!(unit_maps[2], FunctionMapping::default()); + assert!(!integration_maps[2].lines.is_empty()); + // Outside the focal range is absent even when the file has coverage. + assert_eq!( + unit.mapping(&work, "baseline_fixture::absent", &file, 100, 110) + .unwrap(), + FunctionMapping::default() + ); + + // Compare the ordinary-library object maps with a real integration run, + // including generic instantiations and cfg(not(test)) code. + let calls = work.join("tests/calls.rs"); + fs::write(&calls, "use baseline_fixture::*;\n#[test]\nfn calls() { assert_eq!(ordinary(1), 1); assert_eq!(conditional(1), 1); assert_eq!(production_only(1), 1); assert_eq!(generic(1, 2), 2); assert_eq!(*Container(3).get(), 3); }\n").unwrap(); + collect_coverage_for_tests(&work, true, &["calls".into()]).unwrap(); + let real_lcov = checked( + Command::new("cargo") + .args(["llvm-cov", "report", "--lcov"]) + .current_dir(&work), + ) + .unwrap(); + let real = parse_report( + &String::from_utf8(real_lcov.stdout).unwrap(), + &fs::read_to_string(work.join("coverage.json")).unwrap(), + ) + .unwrap(); + for (marker, expected) in markers.into_iter().zip(&integration_maps) { + let (begin, end) = range(marker); + assert_eq!(&real.function(&file, begin, end), expected, "{marker}"); + } + // A subsequent baseline must not add profiles to the real run. + let before = fs::read_to_string(work.join("coverage.json")).unwrap(); + let mut fresh = BaselineCache::new(&work, true).unwrap(); + fresh + .mapping(&work, "baseline_fixture::ordinary", &file, 1, 3) + .unwrap(); + checked( + Command::new("cargo") + .args([ + "llvm-cov", + "report", + "--json", + "--output-path", + "after.json", + ]) + .current_dir(&work), + ) + .unwrap(); + assert_eq!(before, fs::read_to_string(work.join("after.json")).unwrap()); + + // Build errors restore temporary source annotations and the helper file. + let broken = format!("{SOURCE}\ncompile_error!(\"baseline build failure\");\n"); + fs::write(&file, &broken).unwrap(); + let mut failing = BaselineCache::new(&work, true).unwrap(); + let error = failing + .mapping(&work, "baseline_fixture::ordinary", &file, 1, 3) + .unwrap_err(); + assert!(error.to_string().contains("baseline build failure")); + assert_eq!(fs::read_to_string(&file).unwrap(), broken); + assert_eq!( + fs::read_to_string(&helper).unwrap(), + "// existing file must survive\n" + ); + // Existing cache entries do not rebuild on each focal function. + assert_eq!( + unit.mapping(&work, "baseline_fixture::ordinary", &file, 1, 3) + .unwrap(), + unit_maps[0] + ); + + // A standalone binary uses its test harness, never executes main, and + // rejects function-level integration mode before needing a baseline. + let bin = work.join("binary"); + let bin_source = format!("{SOURCE}\nfn main() {{ panic!(\"baseline executed main\"); }}\n"); + let bin_file = fixture(&bin, &bin_source, true); + let mut binary = BaselineCache::new(&bin, false).unwrap(); + assert_eq!( + binary + .mapping(&bin, "baseline_fixture::ordinary", &bin_file, 1, 3) + .unwrap(), + unit_maps[0] + ); + assert_eq!(fs::read_to_string(&bin_file).unwrap(), bin_source); + assert!( + select_target( + &binary.metadata, + "baseline_fixture::ordinary", + &bin_file, + true + ) + .unwrap_err() + .to_string() + .contains("require a library") + ); + + // Same-name library and binary modules cannot be selected by crate name. + fs::write(work.join("src/main.rs"), "fn main() {}\n").unwrap(); + let mixed = project_metadata(&work).unwrap(); + assert!( + select_target( + &mixed, + "baseline_fixture::shared::f", + &work.join("src/shared.rs"), + false + ) + .is_err() + ); + assert!( + select_target(&mixed, "baseline_fixture::ordinary", &file, false) + .unwrap() + .is_kind(TargetKind::Lib) + ); + + let cfg_only = work.join("cfg-only"); + let cfg_file = fixture( + &cfg_only, + "#[cfg(not(test))]\npub fn production_only() -> i32 { 1 }\n", + false, + ); + let mut absent = BaselineCache::new(&cfg_only, false).unwrap(); + assert_eq!( + absent + .mapping( + &cfg_only, + "baseline_fixture::production_only", + &cfg_file, + 1, + 2 + ) + .unwrap(), + FunctionMapping::default() + ); +} diff --git a/utgen/src/run/llm_fix.rs b/utgen/src/run/llm_fix.rs index 8ee548561e..88ef44323b 100644 --- a/utgen/src/run/llm_fix.rs +++ b/utgen/src/run/llm_fix.rs @@ -594,6 +594,7 @@ pub async fn llm_fix( "--tasks is outside the supported positive range", )); } + super::validate_targets(&project_path, &work_path, functions, integration)?; let parent_dir = project_path.join("utgen/generation/llm_fix"); let map_path = work_path.join("brinfo/name_map.json"); let nmap: HashMap = diff --git a/utgen/src/run/mod.rs b/utgen/src/run/mod.rs index c306d44e8f..e105d6790b 100644 --- a/utgen/src/run/mod.rs +++ b/utgen/src/run/mod.rs @@ -1,5 +1,6 @@ static TIMEOUT_DERIVE: &str = "#[timeout(5000)]"; mod coverage; +mod coverage_baseline; mod coverage_json; pub(crate) mod integration; mod llm_fix; @@ -15,6 +16,7 @@ pub use run::gen_test_rate; pub use run::gen_test_rate_aggregated; pub use coverage::collect_coverage; +pub(crate) use coverage_baseline::validate_targets; /// Assemble a statistics test with the shared execution limit. fn timed_test(signature: &str, attrs: &[String], body: &[String]) -> Vec { diff --git a/utgen/src/run/run.rs b/utgen/src/run/run.rs index aec5f1254a..9831203075 100644 --- a/utgen/src/run/run.rs +++ b/utgen/src/run/run.rs @@ -1,4 +1,5 @@ use super::{ + coverage_baseline::{BaselineCache, FunctionMapping}, coverage_json::CoverageJson, integration::{self, gen_integration}, prepare::get_test_gen_infos, @@ -7,7 +8,7 @@ use crate::{ FunctionSelection, run::timed_test, types::TestGenInfo, - utils::{backup_file, delete_backup, insert_test, restore_file}, + utils::{RestoreOnDrop, backup_file, delete_backup, insert_test, restore_file}, run::run_all::{prepare_all_unit_tests_for_one_shot_run}, }; use log::{error, info}; @@ -177,6 +178,28 @@ pub struct TestRateInfo { } impl TestRateInfo { + // A baseline supplies only the denominator, never a generated candidate or hit. + fn apply_baseline(&mut self, mapping: FunctionMapping) { + self.codes_lines = mapping.lines; + self.codes_branches = mapping.branches.into_iter().map( + |(start_line, start_column, end_line, end_column)| BranchCoverageInfo { + start_line, + start_column, + end_line, + end_column, + positive: false, + negative: false, + }, + ).collect(); + self.coverage_available = !self.codes_lines.is_empty() || !self.codes_branches.is_empty(); + self.lines = self.codes_lines.len() as i32; + self.branches = if self.codes_branches.is_empty() { + i32::from(self.coverage_available) + } else { + 2 * self.codes_branches.len() as i32 + }; + } + fn parse_codes_lines_and_branches(&mut self) { let mut lines = 0; let mut lines_covered = 0; @@ -363,6 +386,8 @@ pub fn gen_test_rate( } } else { let integration_infos = gen_integration(&test_gen_infos, work_dir); + let mut baselines = + BaselineCache::new(work_dir, true).expect("Cannot prepare coverage baselines"); let mut test_rate_infos: Vec = Vec::new(); let start_time = SystemTime::now(); // let _ = target_clean(work_dir); @@ -447,6 +472,12 @@ pub fn gen_test_rate( &mut test_rate_info, ); } + if !test_rate_info.coverage_available { + let mapping = baselines + .mapping(work_dir, &function_name, &file_path, begin, end) + .expect("Cannot collect integration coverage baseline"); + test_rate_info.apply_baseline(mapping); + } test_rate_infos.push(test_rate_info); break; } @@ -780,6 +811,7 @@ fn gen_coverage_and_pass_rate( test_rate_infos: &mut Vec, ) { let mut progress = 0; + let mut baselines = BaselineCache::new(work_dir, false).expect("Cannot prepare coverage baselines"); for test_gen_info in test_gen_infos.iter() { let file_rela = test_gen_info.get_file(); let file_path = project_dir.join(&file_rela); @@ -801,6 +833,7 @@ fn gen_coverage_and_pass_rate( let template = include_str!("../../res/code_template.json"); let code_template: Vec = serde_json::from_str(template).unwrap(); backup_file(Path::new(&file_path)); + let restore = RestoreOnDrop(&file_path); let mut test_rate_info = TestRateInfo { function_name: name.clone(), @@ -948,46 +981,7 @@ fn gen_coverage_and_pass_rate( } } } - if tests_run == 0 { - let mut mod_code = code_template.clone(); - let sig = format!("fn test_{}()", fn_name,); - info!("Running test fn test_{}_{:02}()", fn_name, tests); - let mut fn_code = vec!["#[test]".to_string()]; - fn_code.push(sig); - let empty_codes = vec!["{".to_string(), "}".to_string()]; - fn_code.extend(empty_codes.clone()); - let pos = mod_code.len() - 1; - mod_code.splice(pos..pos, fn_code); - insert_test(insert_kind, Path::new(&file_path), &mod_code); - // let _ = target_clean(work_dir); - let test_type = TestType::CoverageRate; - let test_name = format!("test_{}", fn_name); - let run_test_output = run_test(project_dir, work_dir, test_type, true, false); - let loc = test_gen_info.get_loc(); - let begin = loc.get_startline() as i32; - let end = loc.get_endline() as i32; - gen_one_codes_lines( - project_dir, - work_dir, - &name, - &file_path, - begin, - end, - &empty_codes, - &mut test_rate_info, - ); - gen_one_codes_branchs( - project_dir, - work_dir, - &name, - &file_path, - begin, - end, - &empty_codes, - &mut test_rate_info, - ); - restore_file(&file_path); - } + drop(restore); delete_backup(&file_path); tests_compiled_rate = if tests > 0 { tests_compiled as f64 / tests as f64 * 100.0 @@ -1024,6 +1018,18 @@ fn gen_coverage_and_pass_rate( test_rate_info.oracles_passed = oracles_passed; test_rate_info.oracles_passed_rate = oracles_passed_rate; test_rate_info.parse_codes_lines_and_branches(); + if !test_rate_info.coverage_available { + let loc = test_gen_info.get_loc(); + let mapping = baselines.mapping( + work_dir, + &name, + &file_path, + loc.get_startline() as i32, + loc.get_endline() as i32, + ) + .expect("Cannot collect unit coverage baseline"); + test_rate_info.apply_baseline(mapping); + } test_rate_infos.push(test_rate_info); } }