diff --git a/docs/minimal-pipeline.md b/docs/minimal-pipeline.md index 494e8d6c44..857da26814 100644 --- a/docs/minimal-pipeline.md +++ b/docs/minimal-pipeline.md @@ -41,6 +41,7 @@ The default binary directory is `target/debug`. The coverage tool must be availa | Coverage | Excludes test bodies and test-only helpers, preserves production coverage, and executes tests once when exporting XML and JSON. | | Cleanup | Restores sources after the covered success, assertion-failure, compilation-failure, and report-export-failure cases. | | Candidate timeout | Ordinary and `should_panic` infinite loops count as failures; an expected panic and a later ordinary test pass in unit and integration statistics. | +| Statistics | Oracle groups count once across multiple candidates; pass results match complete test names; an all-compilation-failure result reports unavailable coverage without exporting an empty run. | | Integration repair | Repairs alias/trait imports, helpers, and test bodies in integration targets; keeps sibling candidates independent; checks cached repair, mode mismatch, unselected targets, and cleanup after request failure. | | Integration directories | Original and generated tests coexist after normal runs and cached reruns; original-test compilation failure restores both directories; an existing staging directory is preserved and reported. | diff --git a/docs/palm-rust-unit-test-generation.md b/docs/palm-rust-unit-test-generation.md index 5866aec472..e8f5d7f18f 100644 --- a/docs/palm-rust-unit-test-generation.md +++ b/docs/palm-rust-unit-test-generation.md @@ -422,7 +422,7 @@ export PALM_CONFIG="$(pwd)/utgen/res/api.json" | `coverage.xml` | `cargo llvm-cov` Cobertura 中间输出，位于目标 work dir，解析后可能被删除。 | | `coverage.json` | `cargo llvm-cov` JSON 中间输出，位于目标 work dir，解析后可能被删除。 | -最终统计以 `utgen/result/` 和 `utgen/fixed_result/` 中的 JSON 为准；当前实现不生成 HTML 报告。 +最终统计以 `utgen/result/` 和 `utgen/fixed_result/` 中的 JSON 为准；当前实现不生成 HTML 报告。`coverage_available=false` 表示没有取得被测函数的覆盖映射，覆盖数值仅为占位，不应参与汇总，也不能直接解释为条件编译排除了函数。旧结果缺少该字段时，其可用性未确认。`tests_*` 按候选计数，`oracles_*` 按原有 TestInfo 分组计数；一组中有候选通过，该组即计为通过。运行数保留包含 ignored 条目的既有口径。 ### 8.7 LLM 修复流程 diff --git a/scripts/check_minimal.py b/scripts/check_minimal.py index f05e3e2804..23455cf698 100644 --- a/scripts/check_minimal.py +++ b/scripts/check_minimal.py @@ -356,6 +356,8 @@ def do_POST(self): result = json.loads((target / f'utgen/result/{names[selected]}.json').read_text()) assert result['tests'] == result['tests_compiled'] == result['tests_run'] == 4, result assert result['tests_passed'] == 2, result + assert result['oracles_run'] == 4 and result['oracles_passed'] == 2, result + assert result['coverage_available'], result assert result['branches_covered'] == result['branches'] == 2, result if integration: assert not (target / 'tests.bak2').exists() @@ -368,6 +370,38 @@ def do_POST(self): shutil.move(original_tests, target / 'tests.bak') assert snapshot(target) == prepared + # Multiple candidates belong to one oracle; all failed compilation is + # still a valid result and must not attempt to export empty coverage. + data['integration'] = True + answer['common'] = [] + answer['chain_tests'] = [dict( + attrs=[], prefix=[], oracles=[], + codes=[['{', 'assert_eq!(classify(2), 1);', '}'], + ['{', 'assert_eq!(classify(-1), 1);', '}']], + can_compile=[{'Ok': None}, {'Ok': None}], repaired=[False, False])] + candidate.write_text(json.dumps(data)) + command = ['utgen', 'gen', '-p', str(target), '--functions-file', str(selection), '--integration'] + run('integration-oracle-group', command) + result = json.loads((target / f'utgen/result/{names[selected]}.json').read_text()) + assert (result['tests_run'], result['tests_passed']) == (2, 1), result + assert (result['oracles'], result['oracles_compiled'], result['oracles_run'], result['oracles_passed']) == (1, 1, 1, 1), result + assert result['oracles_passed_rate'] == 100.0 and result['coverage_available'], result + test = answer['chain_tests'][0] + test['codes'] = [['{', 'missing();', '}']] + test['can_compile'] = [{'Err': 'fixture compile failure'}] + test['repaired'] = [False] + candidate.write_text(json.dumps(data)) + run('integration-no-compiled-candidates', command) + result = json.loads((target / f'utgen/result/{names[selected]}.json').read_text()) + assert result['tests'] == result['oracles'] == 1, result + assert result['tests_compiled'] == result['tests_run'] == result['tests_passed'] == 0, result + assert result['oracles_compiled'] == result['oracles_run'] == result['oracles_passed'] == 0, result + assert not result['coverage_available'], result + assert result['lines'] == result['branches'] == 0, result + assert result['codes_lines_covered'] == [], result + shutil.rmtree(target / 'tests') + assert snapshot(target) == prepared + # Original and generated integration tests must coexist, including on # a cached rerun and after original-test compilation fails. answer['chain_tests'] = [dict( @@ -460,7 +494,8 @@ def integration_snapshot(): result = json.loads((target / f'utgen/fixed_result/{names[name]}.json').read_text()) assert result['tests'] == result['tests_compiled'] == result['tests_run'] == result['tests_passed'] == count, result assert len(result['codes_lines_covered']) == count, result - assert result['lines_covered'] > 0, result + assert result['lines_covered'] > 0 and result['coverage_available'], result + assert result['oracles_run'] == result['oracles_passed'] == count, result fixed_path = target / f'utgen/generation/llm_fix/{names[selected]}.json' fixed = json.loads(fixed_path.read_text()) tests = fixed['fn_tests'][0]['answers'][0]['chain_tests'] diff --git a/utgen/README.md b/utgen/README.md index eee84c01a9..f7c01a7ad2 100644 --- a/utgen/README.md +++ b/utgen/README.md @@ -216,6 +216,10 @@ Paths under `utgen/` below are relative to `--project-dir`: | `utgen/fixed_result/` | Post-repair coverage and execution statistics per focal function. | | `utgen/original_result.json` | Comparison statistics when the original integration-test backup is available. | +Per-function results include `coverage_available`. When it is false, the numeric coverage fields are placeholders and must not enter coverage aggregates; report the number of unavailable functions separately. Missing mappings are not assumed to mean `cfg(test)` exclusion. Old results without this field have unconfirmed availability. Candidate/compilation statistics remain valid when no candidate compiles. + +`tests_*` count candidates; `oracles_*` count the existing `TestInfo` groups. A group counts once if any of its candidates compiles, is registered to run, or passes, respectively. Run counts retain the existing libtest convention, including ignored tests. Integration pass outcomes match complete test names. + `coverage.xml`, `coverage.json`, and `error_output.json` are intermediate files in the work directory and may be deleted after generation/repair parses them. The standalone `coverage` command retains both reports. The current implementation does not produce an HTML report. See the [deterministic minimal pipeline](../docs/minimal-pipeline.md) and the [bytes example](../examples/README.md) for a working-copy workflow and the [Chinese technical guide](../docs/palm-rust-unit-test-generation.md) for implementation details. diff --git a/utgen/src/run/integration.rs b/utgen/src/run/integration.rs index 5167050bac..5adcad6ed1 100644 --- a/utgen/src/run/integration.rs +++ b/utgen/src/run/integration.rs @@ -168,7 +168,7 @@ pub(crate) fn repaired_candidate( pub struct IntegrationInfo { pub function_name: String, pub file_name: String, - pub test_function_names: Vec, + pub test_functions: Vec<(String, usize)>, // (complete libtest name, oracle group) pub tests: i32, pub tests_lines: Vec, pub oracles: i32, @@ -179,11 +179,39 @@ pub struct IntegrationInfo { } impl IntegrationInfo { + /// Keep the existing run-count convention (registered tests, including ignored). + /// Count a group once if any of its candidates was registered/passed. + pub fn execution_counts(&self, output: &[String]) -> (i32, i32, i32, i32) { + let passed: HashSet<_> = output + .iter() + .filter_map(|line| { + let (name, outcome) = line.strip_prefix("test ")?.split_once(" ... ")?; + (outcome == "ok").then_some(name) + }) + .collect(); + let mut run_groups = BTreeSet::new(); + let mut passed_groups = BTreeSet::new(); + let mut passed_tests = 0; + for (name, group) in &self.test_functions { + run_groups.insert(group); + if passed.contains(name.as_str()) { + passed_tests += 1; + passed_groups.insert(group); + } + } + ( + self.test_functions.len() as i32, + passed_tests, + run_groups.len() as i32, + passed_groups.len() as i32, + ) + } + fn new() -> Self { IntegrationInfo { function_name: String::new(), file_name: String::new(), - test_function_names: Vec::new(), + test_functions: Vec::new(), tests: 0, tests_lines: Vec::new(), oracles: 0, @@ -234,7 +262,7 @@ pub fn gen_integration(test_gen_infos: &Vec, work_dir: &Path) -> Ve &test_name, true, )); - result.test_function_names.push(test_name); + result.test_functions.push((format!("candidate_{id}::{test_name}"), result.oracles as usize - 1)); result.tests_compiled += 1; } } @@ -262,6 +290,28 @@ pub fn gen_integration(test_gen_infos: &Vec, work_dir: &Path) -> Ve mod tests { use super::*; + #[test] + fn execution_counts_use_complete_names_and_distinct_oracle_groups() { + let mut info = IntegrationInfo::new(); + info.test_functions = vec![ + ("candidate_10::test_10".into(), 0), + ("candidate_100::test_100".into(), 0), + ("candidate_2::test_2".into(), 1), + ]; + let output = [ + "test candidate_100::test_100 ... ok", + "test candidate_10::test_10 ... FAILED", + "test candidate_2::test_2 ... ignored", + "test unrelated::test_10 ... ok", + ] + .map(str::to_owned); + assert_eq!(info.execution_counts(&output), (3, 1, 2, 1)); + assert_eq!( + IntegrationInfo::new().execution_counts(&output), + (0, 0, 0, 0) + ); + } + #[test] fn repair_cannot_disable_the_candidate_to_hide_a_compile_error() { let context = IntegrationContext { diff --git a/utgen/src/run/run.rs b/utgen/src/run/run.rs index a586590731..aec5f1254a 100644 --- a/utgen/src/run/run.rs +++ b/utgen/src/run/run.rs @@ -162,6 +162,8 @@ pub struct TestRateInfo { pub tests_run: i32, pub tests_passed: i32, pub tests_passed_rate: f64, + #[serde(default)] + pub coverage_available: bool, pub lines: i32, pub lines_covered: i32, pub lines_coveraged_rate: f64, @@ -219,10 +221,11 @@ impl TestRateInfo { } } } + self.coverage_available = lines > 0 || !codes_branches_covered.is_empty(); branches = if codes_branches_covered.len() > 0 { 2 * codes_branches_covered.len() as i32 } else { - 1 + i32::from(self.coverage_available) }; branches_covered = if codes_branches_covered.len() > 0 { let mut temp_branches_covered = 0; @@ -368,14 +371,19 @@ pub fn gen_test_rate( } let targets = integration_infos .iter() + .filter(|info| info.tests_compiled > 0) .map(|info| info.file_name.clone()) .collect::>(); - let output = super::coverage::collect_coverage_for_tests(work_dir, true, &targets) - .expect("Failed to collect integration coverage"); - let test_output = String::from_utf8_lossy(&output.stdout) - .lines() - .map(str::to_owned) - .collect::>(); + let test_output = if targets.is_empty() { + Vec::new() + } else { + let output = super::coverage::collect_coverage_for_tests(work_dir, true, &targets) + .expect("Failed to collect integration coverage"); + String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_owned) + .collect() + }; let end_time = SystemTime::now(); let test_time = end_time.duration_since(start_time).unwrap().as_secs_f64(); for integration_info in integration_infos.iter() { @@ -401,6 +409,7 @@ pub fn gen_test_rate( tests_run: 0, tests_passed: 0, tests_passed_rate: 0.0, + coverage_available: false, lines: 0, lines_covered: 0, lines_coveraged_rate: 0.0, @@ -418,62 +427,53 @@ pub fn gen_test_rate( let file_path = project_dir.join(&file_rela); let begin = loc.get_startline() as i32; let end = loc.get_endline() as i32; - gen_one_coverage_rate_lines( - project_dir, - work_dir, - &function_name, - &file_path, - begin, - end, - &mut test_rate_info, - ); - gen_one_coverage_rate_branches( - project_dir, - work_dir, - &function_name, - &file_path, - begin, - end, - &mut test_rate_info, - ); + if !targets.is_empty() { + gen_one_coverage_rate_lines( + project_dir, + work_dir, + &function_name, + &file_path, + begin, + end, + &mut test_rate_info, + ); + gen_one_coverage_rate_branches( + project_dir, + work_dir, + &function_name, + &file_path, + begin, + end, + &mut test_rate_info, + ); + } test_rate_infos.push(test_rate_info); break; } } } // Every integration function reads the same coverage run. - fs::remove_file(work_dir.join("coverage.xml")).unwrap(); - fs::remove_file(work_dir.join("coverage.json")).unwrap(); - for integration_info in integration_infos.iter() { - let mut tests = integration_info.test_function_names.len() as i32; - let mut passed_tests = 0; - for test_function_name in integration_info.test_function_names.iter() { - for line in test_output.iter() { - if line.contains(test_function_name) { - info!("Generate pass rate for {}", test_function_name); - let line_spilit = line - .split(" ... ") - .map(|line| line.to_string()) - .collect::>(); - if line_spilit[1] == "ok" { - passed_tests = passed_tests + 1; - } - break; - } - } - } - for already_test_rate_info in test_rate_infos.iter_mut() { - if already_test_rate_info.function_name == integration_info.function_name { - already_test_rate_info.tests_run = tests; - already_test_rate_info.tests_passed = passed_tests; - already_test_rate_info.tests_passed_rate = if tests > 0 { - passed_tests as f64 / tests as f64 * 100.0 - } else { - 0.0 - }; - break; - } - } + if !targets.is_empty() { + fs::remove_file(work_dir.join("coverage.xml")).unwrap(); + fs::remove_file(work_dir.join("coverage.json")).unwrap(); + } + for (info, rate) in integration_infos.iter().zip(&mut test_rate_infos) { + let (tests_run, tests_passed, oracles_run, oracles_passed) = + info.execution_counts(&test_output); + rate.tests_run = tests_run; + rate.tests_passed = tests_passed; + rate.oracles_run = oracles_run; + rate.oracles_passed = oracles_passed; + rate.tests_passed_rate = if tests_run > 0 { + tests_passed as f64 / tests_run as f64 * 100.0 + } else { + 0.0 + }; + rate.oracles_passed_rate = if oracles_run > 0 { + oracles_passed as f64 / oracles_run as f64 * 100.0 + } else { + 0.0 + }; } // let _ = target_clean(work_dir); gen_codes_lines_and_branches_covered( @@ -497,7 +497,9 @@ pub fn gen_test_rate( if is_pre { gen_coverage_rate_for_original_tests(project_dir, work_dir, &test_gen_infos); } - generated_tests.finish().expect("cannot restore generated tests"); + generated_tests + .finish() + .expect("cannot restore generated tests"); } } @@ -630,6 +632,7 @@ fn gen_one_coverage_rate_lines( // 0.0 // }; + test_rate_info.coverage_available = lines > 0; test_rate_info.lines = lines; test_rate_info.lines_covered = lines_covered; test_rate_info.lines_coveraged_rate = lines_coverage_rate; @@ -700,6 +703,7 @@ fn gen_one_coverage_rate_branches( let mut branches_covered = 0; let mut branches_coverage_rate = 0.0; + test_rate_info.coverage_available |= !codes_branches.is_empty(); if codes_branches.len() > 0 { branches = 2 * codes_branches.len() as i32; for codes_branch in codes_branches.iter() { @@ -716,7 +720,7 @@ fn gen_one_coverage_rate_branches( 0.0 }; } else { - branches = 1; + branches = i32::from(test_rate_info.coverage_available); if test_rate_info.lines_covered > 0 { branches_covered = 1; } @@ -815,6 +819,7 @@ fn gen_coverage_and_pass_rate( tests_run: 0, tests_passed: 0, tests_passed_rate: 0.0, + coverage_available: false, lines: 0, lines_covered: 0, lines_coveraged_rate: 0.0,