diff --git a/README.md b/README.md index 5c8036bf..3c14073c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ For Go, `ddtest instrument` reports a project's tests to Test Optimization directly; see [Go tests with ddtest instrument](#go-tests-with-ddtest-instrument). `ddtest onboard` and `ddtest testdrive` also support Java projects built with -Maven; JUnit 4, JUnit 5, TestNG, Spock and Cucumber JVM get paired validation. `ddtest plan` and `ddtest run` +Maven; JUnit 4, JUnit 5, TestNG, Spock, Cucumber JVM, Karate, ScalaTest and MUnit get paired validation. `ddtest plan` and `ddtest run` do not support Java, because the Java agent performs Test Impact Analysis inside the Maven build. See [Java validation](docs/onboarding.md#java-validation). diff --git a/docs/onboarding.md b/docs/onboarding.md index 50a1fe02..0fbb4e50 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -455,21 +455,32 @@ select the framework the project uses (`junit4`, `junit5`, `testng`, `spock`, contains a JavaScript, Python or Ruby project keeps that platform by default; select Java with `--platform java` or a Maven `--framework`. -JUnit 4, JUnit 5, TestNG, Spock and Cucumber JVM support plain `ddtest testdrive --all --check-only --yes` +Every Maven framework supports plain `ddtest testdrive --all --check-only --yes` and full `ddtest testdrive --all --yes` validation, or a focused `ddtest testdrive --framework junit5 --command './mvnw -B verify' --yes`. Each configuration runs in a disposable copy with fresh build output, without and with -dd-java-agent. Native Surefire/Failsafe results must match (identities, outcomes, +dd-java-agent. Native results (Surefire/Failsafe reports, the +scalatest-maven-plugin's `target/scalatest-reports` and Karate's +`target/karate-reports` feature reports) must match (identities, outcomes, failure types and messages, exit code), each must match one reported test execution, the reporting run must cover repository application sources, and all six feature controls run with a probe test in a module of the selected framework -(a Groovy specification for Spock, compiled by the project's GMavenPlus setup). +(a Groovy specification for Spock, compiled by the project's GMavenPlus setup, +and a Scala suite for ScalaTest and MUnit, compiled by its scala-maven-plugin). For Cucumber JVM the probe is a feature with a step in the glue package of the module's runner class (`@RunWith(Cucumber.class)` or a JUnit Platform suite of the `cucumber` engine), so the project's own glue, hooks and object factory load; the runner is selected with the probe feature in place of the project's. A module without such a runner, or a suite that pins its own features or tags, -leaves the features inconclusive. Probe builds stop at the `test` phase, so +leaves the features inconclusive. For Karate the probe is a feature that the +module's runner class (`@Karate.Test`, `@RunWith(Karate.class)` or a JUnit test +calling `Runner.path`) runs through `karate.options`, so `karate-config.js` and +`karate.env` still apply; the runner's own JUnit result is left out. MUnit's +runner is not one dd-java-agent can skip tests in, so its skipping and disabled +features are reported as not supported. A ScalaTest suite that Surefire runs +through ScalaTest's `JUnitRunner` gets a probe with the same runner; dd-java-agent +1.67.1 and 1.68.0 fail every test of such a suite, which validation reports as a +suspected regression. Probe builds stop at the `test` phase, so checks after the tests, such as license headers, do not judge the probe. Matching pre-existing failures are permitted; a build that stops for anything other than native test failures stays inconclusive. @@ -495,7 +506,6 @@ major is validated on the local JDK as a substitute, recorded in a decision for the user, and is then reported as a caveat. Validate that JDK instead by running testdrive with `JAVA_HOME` set to it. `ddtest plan` and `ddtest run` do not support Java: the agent performs Test Impact Analysis inside the Maven build. -Karate, ScalaTest and MUnit run reporting-only checks. Coding agents start with `ddtest onboard --agent`, as for Ruby: there is no automatic Maven CI edit. Until every discovered test step of the framework runs after a diff --git a/internal/maven/reports.go b/internal/maven/reports.go index 60bb8c26..87543c44 100644 --- a/internal/maven/reports.go +++ b/internal/maven/reports.go @@ -7,6 +7,7 @@ package maven import ( "cmp" + "encoding/json" "encoding/xml" "errors" "fmt" @@ -17,7 +18,8 @@ import ( "strings" ) -// Test is one test case from a module's native Surefire or Failsafe report. +// Test is one test case from a module's native Surefire, Failsafe, ScalaTest +// or Karate report. type Test struct { // Module is the project directory relative to the inspected root. Module, Suite, Name, Status string @@ -30,7 +32,7 @@ type Test struct { var ErrNoTestCases = errors.New("native reports contain no test cases") // ErrNoReports reports a build that wrote no native report for any module. -var ErrNoReports = errors.New("no native TEST-*.xml reports under the modules' target/surefire-reports or target/failsafe-reports; configure Surefire or Failsafe XML reporting and run the intended Maven profile") +var ErrNoReports = errors.New("no native reports under the modules' target/surefire-reports, target/failsafe-reports, target/scalatest-reports or target/karate-reports; configure Surefire or Failsafe XML reporting (the scalatest-maven-plugin writes it with .) and run the intended Maven profile") type reportCase struct { Class string `xml:"classname,attr"` @@ -51,8 +53,12 @@ type reportSuite struct { Cases []reportCase `xml:"testcase"` } -// reportDirectories are where Surefire and Failsafe write one TEST-*.xml per class. -var reportDirectories = []string{filepath.Join("target", "surefire-reports"), filepath.Join("target", "failsafe-reports")} +// reportDirectories are where Surefire, Failsafe and the scalatest-maven-plugin +// write one TEST-*.xml per class. +var reportDirectories = []string{filepath.Join("target", "surefire-reports"), filepath.Join("target", "failsafe-reports"), filepath.Join("target", "scalatest-reports")} + +// karateReports is where Karate writes one *.karate-json.txt per feature. +var karateReports = filepath.Join("target", "karate-reports") // ReadReports reads the native reports of every project under root. Call it // only on fresh build output: reports from an earlier build are read as well. @@ -60,28 +66,49 @@ func ReadReports(root string, projects []Project) ([]Test, error) { var tests []Test reports := 0 for _, project := range projects { + var paths []string for _, directory := range reportDirectories { - paths, err := filepath.Glob(filepath.Join(root, project.Directory, directory, "TEST-*.xml")) + found, err := filepath.Glob(filepath.Join(root, project.Directory, directory, "TEST-*.xml")) if err != nil { return nil, err } - slices.Sort(paths) - for _, path := range paths { - info, err := os.Lstat(path) - if err != nil { - return nil, err - } - if !info.Mode().IsRegular() { - return nil, fmt.Errorf("native test report is not a regular file: %s", path) - } - read, err := readReport(path, project.Directory) - if err != nil { - return nil, err - } - reports++ - tests = append(tests, read...) + slices.Sort(found) + paths = append(paths, found...) + } + features, err := filepath.Glob(filepath.Join(root, project.Directory, karateReports, "*.karate-json.txt")) + if err != nil { + return nil, err + } + slices.Sort(features) + var module, karate []Test + var karatePaths []string + for _, path := range append(paths, features...) { + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("native test report is not a regular file: %s", path) } + var read []Test + if strings.HasSuffix(path, ".karate-json.txt") { + var feature string + read, feature, err = readKarateReport(path, project.Directory) + karate, karatePaths = append(karate, read...), append(karatePaths, feature) + } else { + read, err = readReport(path, project.Directory) + module = append(module, read...) + } + if err != nil { + return nil, err + } + reports++ } + // Surefire reports each Karate scenario of a JUnit 5 runner under its + // feature path, as example/calc, and may lose some; the feature's + // own report replaces them. + module = slices.DeleteFunc(module, func(test Test) bool { return slices.Contains(karatePaths, test.Suite) }) + tests = append(append(tests, module...), karate...) } switch { case reports == 0: @@ -92,6 +119,59 @@ func ReadReports(root string, projects []Project) ([]Test, error) { return tests, nil } +// karateFeature is the part of a Karate feature's JSON report that names its +// scenarios and their outcomes. +type karateFeature struct { + Name string `json:"name"` + RelativePath string `json:"relativePath"` + CallDepth int `json:"callDepth"` + Scenarios []struct { + Name string `json:"name"` + RefID string `json:"refId"` + Failed bool `json:"failed"` + Error string `json:"error"` + Steps []json.RawMessage `json:"stepResults"` + } `json:"scenarioResults"` +} + +// readKarateReport reads a feature's Karate JSON report. Each scenario is a +// test of the suite dd-java-agent reports, "[] ", where feature +// is the feature's path without .feature, which readKarateReport also returns. +// As in dd-java-agent, an unnamed scenario is named by its reference, as +// [2:6], and one that ran no step, as a hook skips it, is skipped. +func readKarateReport(path, module string) ([]Test, string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, "", err + } + var report karateFeature + if err := json.Unmarshal(data, &report); err != nil { + return nil, "", fmt.Errorf("parse native report %s: %w", path, err) + } + if report.RelativePath == "" { + return nil, "", fmt.Errorf("native report %s names no feature path", path) + } + feature := strings.TrimSuffix(report.RelativePath, ".feature") + // A feature another one calls is part of the caller's scenario. + if report.CallDepth > 0 { + return nil, feature, nil + } + var tests []Test + for _, scenario := range report.Scenarios { + test := Test{Module: module, Suite: "[" + feature + "] " + report.Name, Name: cmp.Or(strings.TrimSpace(scenario.Name), scenario.RefID), Status: "passed"} + if len(scenario.Steps) == 0 { + test.Status = "skipped" + } + if scenario.Failed { + // A match failure's first line names only the operator; the + // values that differ follow on the next lines. + test.Status, test.Failure = "failed", "karate: "+strings.Join(strings.Fields(scenario.Error), " ")+"\n" + } + tests = append(tests, test) + } + return tests, feature, nil +} + func readReport(path, module string) ([]Test, error) { data, err := os.ReadFile(path) if err != nil { @@ -146,10 +226,10 @@ func readReport(path, module string) ([]Test, error) { return tests, nil } -var failedGoal = regexp.MustCompile(`^\[ERROR\] Failed to execute goal ([^ ]+) \([^)]*\) on project ([^:]+): (.*)$`) +var failedGoal = regexp.MustCompile(`^\[ERROR\] Failed to execute goal ([^ ]+) \([^)]*\) on project ([^:]+):(.*)$`) // testGoals are the plugin goals whose failure a native test failure explains. -var testGoals = []string{"org.apache.maven.plugins:maven-surefire-plugin:test", "org.apache.maven.plugins:maven-failsafe-plugin:verify"} +var testGoals = []string{"org.apache.maven.plugins:maven-surefire-plugin:test", "org.apache.maven.plugins:maven-failsafe-plugin:verify", "org.scalatest:scalatest-maven-plugin:test"} var ansiEscape = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) @@ -167,13 +247,14 @@ func IncompleteBuild(exitCode int, projects []Project, tests []Test, output stri failedModules[test.Module] = failedModules[test.Module] || test.Status == "failed" } goals := 0 - for _, line := range strings.Split(ansiEscape.ReplaceAllString(output, ""), "\n") { + lines := strings.Split(ansiEscape.ReplaceAllString(output, ""), "\n") + for i, line := range lines { line = strings.TrimSpace(line) if !strings.HasPrefix(line, "[ERROR] Failed to execute goal") { continue } match := failedGoal.FindStringSubmatch(line) - if match == nil || !strings.HasPrefix(match[3], "There are test failures") { + if match == nil || !testFailures(strings.TrimSpace(match[3]), lines[i+1:]) { return true } // The goal is group:artifact:version:goal; the version varies. @@ -190,3 +271,22 @@ func IncompleteBuild(exitCode int, projects []Project, tests []Test, output stri } return exitCode != 0 && goals == 0 } + +// testFailures reports whether a failed goal's message, and the error lines +// after it, describe native test failures. Surefire 3.5 states no message when +// every failed test is an error, and points at the reports on the next lines. +func testFailures(message string, next []string) bool { + if message != "" { + return strings.HasPrefix(message, "There are test failures") + } + for _, line := range next { + detail, ok := strings.CutPrefix(strings.TrimSpace(line), "[ERROR]") + if !ok { + return false + } + if detail = strings.TrimSpace(detail); detail != "" { + return strings.HasPrefix(detail, "See ") && strings.HasSuffix(detail, " for the individual test results.") + } + } + return false +} diff --git a/internal/maven/reports_test.go b/internal/maven/reports_test.go index b44cdfa7..4f03c5c2 100644 --- a/internal/maven/reports_test.go +++ b/internal/maven/reports_test.go @@ -38,11 +38,49 @@ func TestReadReportsReadsSurefireAndFailsafeResults(t *testing.T) { }, tests) } +func TestReadReportsReadsScalaTestAndKarateResults(t *testing.T) { + root := t.TempDir() + projects := []Project{{Directory: "scala"}, {Directory: "api"}} + writePOM(t, root, "scala/target/scalatest-reports/TEST-example.CalcTest.xml", ``) + // The plugin's discovery wrapper holds no test case. + writePOM(t, root, "scala/target/scalatest-reports/TEST-org.scalatest.tools.DiscoverySuite-1.xml", ``) + // Surefire reports a JUnit 5 runner's scenarios by feature path, and here + // loses the failing one; the feature's own report replaces them. + writePOM(t, root, "api/target/surefire-reports/TEST-example.CalcTest.xml", ` + + +`) + writePOM(t, root, "api/target/karate-reports/example.calc.karate-json.txt", `{"name": "Calculator", "relativePath": "example/calc.feature", "callDepth": 0, "scenarioResults": [ + {"name": "adds", "refId": "[1:2]", "failed": false, "stepResults": [{}]}, + {"name": "existing failure", "refId": "[2:6]", "failed": true, "error": "match failed: EQUALS\n 4\n 5", "stepResults": [{}]}, + {"name": "", "refId": "[3:10]", "failed": false, "stepResults": [{}]}, + {"name": "skipped by a hook", "refId": "[4:14]", "failed": false, "stepResults": []} +]}`) + // A called feature is part of its caller's scenario. + writePOM(t, root, "api/target/karate-reports/example.helper.karate-json.txt", `{"name": "Helper", "relativePath": "example/helper.feature", "callDepth": 1, "scenarioResults": [{"name": "login"}]}`) + + tests, err := ReadReports(root, projects) + require.NoError(t, err) + require.Equal(t, []Test{ + {Module: "scala", Suite: "example.CalcTest", Name: "adds", Status: "passed"}, + {Module: "api", Suite: "example.UnitTest", Name: "checks", Status: "passed"}, + {Module: "api", Suite: "[example/calc] Calculator", Name: "adds", Status: "passed"}, + {Module: "api", Suite: "[example/calc] Calculator", Name: "existing failure", Status: "failed", Failure: "karate: match failed: EQUALS 4 5\n"}, + {Module: "api", Suite: "[example/calc] Calculator", Name: "[3:10]", Status: "passed"}, + {Module: "api", Suite: "[example/calc] Calculator", Name: "skipped by a hook", Status: "skipped"}, + }, tests) + + writePOM(t, root, "api/target/karate-reports/example.other.karate-json.txt", `{"name": "Other"}`) + _, err = ReadReports(root, projects) + require.ErrorContains(t, err, "names no feature path") +} + func TestReadReportsDistinguishesMissingAndEmptyResults(t *testing.T) { root := t.TempDir() projects := []Project{{Directory: "."}} _, err := ReadReports(root, projects) require.ErrorIs(t, err, ErrNoReports) + require.ErrorContains(t, err, ".", "the scalatest-maven-plugin writes no XML by default") writePOM(t, root, "target/surefire-reports/TEST-example.EmptyTest.xml", ``) _, err = ReadReports(root, projects) @@ -85,6 +123,9 @@ func TestIncompleteBuild(t *testing.T) { {"successful build", 0, []Test{{Module: "core", Status: "passed"}}, "", false}, {"native test failure", 1, failedCore, testFailure, false}, {"ANSI-colored test failure", 1, failedCore, "\x1b[1;31m" + testFailure + "\x1b[m", false}, + {"ScalaTest plugin failure", 1, failedCore, "[ERROR] Failed to execute goal org.scalatest:scalatest-maven-plugin:2.2.0:test (test) on project core: There are test failures -> [Help 1]\n", false}, + {"Surefire 3.5 test errors only", 1, failedCore, "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:3.5.3:test (default-test) on project core: \n[ERROR] \n[ERROR] See /work/core/target/surefire-reports for the individual test results.\n", false}, + {"failure without a message or reports", 1, failedCore, "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:3.5.3:test (default-test) on project core: \n[ERROR] \n[ERROR] The forked VM terminated without properly saying goodbye.\n", true}, {"Failsafe verification failure", 1, failedCore, "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-failsafe-plugin:3.5.3:verify (default) on project core: There are test failures.\n", false}, {"compilation failure", 1, nil, "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.13.0:compile (default-compile) on project api: Compilation failure\n", true}, {"later module fails to compile", 1, failedCore, testFailure + "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.13.0:compile (default-compile) on project api: Compilation failure\n", true}, diff --git a/internal/testdrive/java_cucumber.go b/internal/testdrive/java_cucumber.go index 3d343e18..5adf3352 100644 --- a/internal/testdrive/java_cucumber.go +++ b/internal/testdrive/java_cucumber.go @@ -112,7 +112,7 @@ func javaCucumberProbe(checkout, module string, args []string) (javaProbe, strin return javaProbe{}, "", err } if runner == "" { - return javaProbe{}, "No Cucumber runner class (@RunWith(Cucumber.class) or a JUnit Platform suite of the cucumber engine) was found in " + module + "'s test sources, so the probe feature cannot be selected alone.", nil + return javaProbe{}, "No Cucumber runner class (@RunWith(Cucumber.class) or a JUnit Platform suite of the cucumber engine) was found in the test sources of " + javaModuleText(module) + ", so the probe feature cannot be selected alone.", nil } // The selected command's own glue property wins over the runner's. for i := 0; i < len(args); i++ { @@ -189,26 +189,6 @@ func javaCucumberRunnerClass(module string) (runner, glue string, suite, pinned return runner, javaGluePackage(glue), suite, pinned, lambda, err } -// javaCucumberModule picks the first module, in reactor order, whose tests ran -// in the baseline and that holds a Cucumber runner class. -func javaCucumberModule(checkout string, projects []maven.Project, tests []nativeTest) (string, bool) { - labels := javaModuleLabels(projects) - executed := map[string]bool{} - for _, test := range tests { - module, _, _ := strings.Cut(test.File, " / ") - executed[module] = true - } - for _, project := range projects { - if !executed[labels[project.Directory]] { - continue - } - if runner, _, _, _, _, err := javaCucumberRunnerClass(filepath.Join(checkout, project.Directory)); err == nil && runner != "" { - return project.Directory, true - } - } - return "", false -} - // javaGluePackage is the first package of a glue list; a path such as // classpath:com/example names the package com.example. func javaGluePackage(glue string) string { diff --git a/internal/testdrive/java_cucumber_test.go b/internal/testdrive/java_cucumber_test.go index 9eaabb8d..213285fe 100644 --- a/internal/testdrive/java_cucumber_test.go +++ b/internal/testdrive/java_cucumber_test.go @@ -155,9 +155,9 @@ func TestJavaCucumberProbeGoesToTheModuleWithTheRunner(t *testing.T) { writeJavaSource(t, filepath.Join(root, "features", "src", "test", "java", "example", "RunCucumberTest.java"), "package example;\n@RunWith(Cucumber.class)\npublic class RunCucumberTest {}\n") projects := []maven.Project{{Directory: ".", Artifact: "parent"}, {Directory: "api", Artifact: "api"}, {Directory: "features", Artifact: "features"}} tests := []nativeTest{{File: "api / example.ApiTest"}, {File: "features / Calculator"}} - module, ok := javaCucumberModule(root, projects, tests) + module, ok := javaRunnerModule(root, projects, tests, "cucumber-java") require.True(t, ok) require.Equal(t, "features", module, "an earlier module without a runner is passed over") - _, ok = javaCucumberModule(root, projects, tests[:1]) + _, ok = javaRunnerModule(root, projects, tests[:1], "cucumber-java") require.False(t, ok, "only modules whose tests ran qualify") } diff --git a/internal/testdrive/java_karate.go b/internal/testdrive/java_karate.go new file mode 100644 index 00000000..8d83d43a --- /dev/null +++ b/internal/testdrive/java_karate.go @@ -0,0 +1,137 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" +) + +// javaKarateFeature is the probe feature. Its scenario records each execution +// and fails as DDTEST_PROBE_MODE asks, as javaProbeSource does, in Karate's +// JavaScript, so no step class is needed. The tag selects it past the tags a +// project's runner filters by. +const javaKarateFeature = `Feature: ` + javaProbeClass + ` + + @ddtest + Scenario: ddtestProbe + * eval + """ + var System = Java.type('java.lang.System'); + var Files = Java.type('java.nio.file.Files'); + var Paths = Java.type('java.nio.file.Paths'); + var FileWriter = Java.type('java.io.FileWriter'); + var path = System.getenv('DDTEST_PROBE_STATE'); + var writer = new FileWriter(path, true); + writer.write('run\n'); + writer.close(); + var attempts = Files.readAllLines(Paths.get(path)).size(); + var mode = System.getenv('DDTEST_PROBE_MODE'); + if (mode == 'fail' || (mode == 'fail-once' && attempts == 1)) karate.fail('ddtest intentional failure'); + """ +` + +// javaKarateProbeFeature is where the probe feature goes, relative to a module. +var javaKarateProbeFeature = filepath.Join("src", "test", "resources", "ddtest", "DdtestValidation.feature") + +// javaKarateRunner is a test class that runs features: a JUnit 5 @Karate.Test +// or a JUnit 4 runner. A class that calls Karate's parallel Runner is one when +// a test calls it. Importing Karate's types does not make a class one. +var javaKarateRunner = regexp.MustCompile(`@(?:[\w.]+\.)?Karate\.Test\b|@(?:[\w.]+\.)?RunWith\(\s*(?:[\w.]+\.)?Karate\.class\s*\)`) + +// javaKarateParallel is a call of Karate's parallel Runner. +var javaKarateParallel = regexp.MustCompile(`\bRunner\.(?:path|builder)\(`) + +// javaKarateMethod is a JUnit 5 @Karate.Test or JUnit @Test method, whose +// name is the first group. +var javaKarateMethod = regexp.MustCompile(`@(?:[\w.]+\.)?(?:Karate\.)?Test\b(?:\([^)]*\))?[^{;=]*?\b([\pL_$][\pL\pN_$]*)\s*\(\s*\)`) + +// javaKarateProbe places the probe feature in module and selects it through +// the module's Karate runner: karate.options replaces the runner's features +// and tags, and keeps its environment and configuration. A runner that runs +// features in parallel from a JUnit test reports that test too; it is the +// probe's harness. Each method of a runner runs the probe, so only one is +// selected. +func javaKarateProbe(checkout, module string) (javaProbe, string, error) { + runner, method, err := javaKarateRunnerClass(filepath.Join(checkout, module)) + if err != nil { + return javaProbe{}, "", err + } + if runner == "" { + return javaProbe{}, "No Karate runner class (@Karate.Test, @RunWith(Karate.class) or Runner.path) was found in the test sources of " + javaModuleText(module) + ", so the probe feature cannot be selected alone.", nil + } + selection := runner + if method != "" { + selection += "#" + method + } + feature := filepath.Join(module, javaKarateProbeFeature) + // The feature is selected by its file: test resources a POM redefines + // may leave it off the classpath. + return javaProbe{ + files: []javaProbeFile{{feature, javaKarateFeature}}, + harness: runner, + args: []string{"-Dtest=" + selection, "-Dkarate.options=--tags @ddtest file:" + filepath.ToSlash(filepath.Join(checkout, feature)), "-Dsurefire.failIfNoSpecifiedTests=false", "-DfailIfNoTests=false", "-DskipITs"}, + }, "", nil +} + +// javaKarateRunnerClass finds the first Karate runner class in a module's test +// sources, by path, and the test method that runs features: its first +// @Karate.Test, else the test that calls the parallel Runner. A JUnit 4 +// runner has none. +func javaKarateRunnerClass(module string) (runner, method string, err error) { + err = filepath.WalkDir(filepath.Join(module, "src", "test", "java"), func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if runner != "" || entry.IsDir() || !strings.HasSuffix(path, ".java") || !entry.Type().IsRegular() { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + text := string(data) + method = javaKarateRunnerMethod(text) + // A helper that calls the Runner outside any test cannot be selected. + if !javaKarateRunner.MatchString(text) && method == "" { + return nil + } + pkg := "" + if match := javaPackage.FindStringSubmatch(text); match != nil { + pkg = match[1] + } + runner = strings.TrimPrefix(pkg+"."+strings.TrimSuffix(entry.Name(), ".java"), ".") + return nil + }) + if errors.Is(err, fs.ErrNotExist) { + return "", "", nil + } + return runner, method, err +} + +// javaKarateRunnerMethod is the test method of a runner's source that runs +// features, as javaKarateRunnerClass describes. +func javaKarateRunnerMethod(text string) string { + matches := javaKarateMethod.FindAllStringSubmatchIndex(text, -1) + for _, match := range matches { + if strings.Contains(text[match[0]:match[1]], "Karate.Test") { + return text[match[2]:match[3]] + } + } + // The test that calls the Runner is the last one declared before the call. + call := javaKarateParallel.FindStringIndex(text) + method := "" + for _, match := range matches { + if call != nil && match[0] < call[0] { + method = text[match[2]:match[3]] + } + } + return method +} diff --git a/internal/testdrive/java_karate_test.go b/internal/testdrive/java_karate_test.go new file mode 100644 index 00000000..2596a801 --- /dev/null +++ b/internal/testdrive/java_karate_test.go @@ -0,0 +1,85 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "path/filepath" + "testing" + + "github.com/DataDog/ddtest/internal/maven" + "github.com/DataDog/ddtest/internal/testdrive/intake" + "github.com/stretchr/testify/require" +) + +func TestJavaKarateProbeSelectsTheFeatureThroughTheRunner(t *testing.T) { + for name, tc := range map[string]struct{ source, selection string }{ + // Every runner method would run the probe; the first is selected. + "JUnit 5 runner": {"package example;\nimport com.intuit.karate.junit5.Karate;\n@SpringBootTest(webEnvironment = WebEnvironment.DEFINED_PORT)\nclass CalcTest {\n @Karate.Test\n Karate calc() { return Karate.run(\"calc\").relativeTo(getClass()); }\n\n @Karate.Test\n Karate other() { return Karate.run(\"other\"); }\n}\n", "example.CalcTest#calc"}, + "qualified JUnit 5 runner": {"package example;\nclass CalcTest {\n @com.intuit.karate.junit5.Karate.Test com.intuit.karate.junit5.Karate calc() { return null; }\n}\n", "example.CalcTest#calc"}, + "JUnit 4 runner": {"package example;\n@RunWith(Karate.class)\n@KarateOptions(features = \"classpath:example\")\npublic class CalcTest {}\n", "example.CalcTest"}, + "parallel runner": {"package example;\nclass CalcTest {\n @Test(timeout = 60000)\n public void all() { Runner.path(\"classpath:example\").tags(\"@smoke\").parallel(4); }\n}\n", "example.CalcTest#all"}, + } { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + writeJavaSource(t, filepath.Join(root, "api", "src", "test", "java", "example", "Calc.java"), "package example;\npublic class Calc {}\n") + writeJavaSource(t, filepath.Join(root, "api", "src", "test", "java", "example", "CalcTest.java"), tc.source) + probe, reason, err := javaProbeFor(root, "api", "karate", nil) + require.NoError(t, err) + require.Empty(t, reason) + require.Equal(t, []javaProbeFile{{filepath.Join("api", javaKarateProbeFeature), javaKarateFeature}}, probe.files) + require.Contains(t, probe.args, "-Dtest="+tc.selection) + require.Equal(t, "example.CalcTest", probe.harness) + require.Contains(t, probe.args, "-Dkarate.options=--tags @ddtest file:"+filepath.ToSlash(filepath.Join(root, "api", javaKarateProbeFeature)), "the tag replaces the runner's tags and the file need not be on the classpath") + }) + } + require.Contains(t, javaKarateFeature, "Feature: "+javaProbeClass) + require.Contains(t, javaKarateFeature, "@ddtest\n Scenario: ddtestProbe") + require.Contains(t, javaKarateFeature, "ddtest intentional failure") +} + +func TestJavaKarateProbeNeedsARunner(t *testing.T) { + root := t.TempDir() + writeJavaSource(t, filepath.Join(root, "src", "test", "java", "example", "CalcTest.java"), "package example;\npublic class CalcTest {}\n") + // A helper that calls the Runner outside any test cannot be selected. + writeJavaSource(t, filepath.Join(root, "src", "test", "java", "example", "AHelper.java"), "package example;\nclass AHelper {\n static Results all() { return Runner.path(\"classpath:example\").parallel(4); }\n}\n") + // A helper that only imports Karate's types runs no feature. + writeJavaSource(t, filepath.Join(root, "src", "test", "java", "example", "Helpers.java"), "package example;\nimport com.intuit.karate.Runner;\nimport com.intuit.karate.junit5.Karate;\npublic class Helpers {}\n") + _, reason, err := javaKarateProbe(root, ".") + require.NoError(t, err) + require.Contains(t, reason, "No Karate runner class") + require.Contains(t, reason, "in the test sources of the root module") + _, reason, err = javaKarateProbe(t.TempDir(), ".") + require.NoError(t, err) + require.Contains(t, reason, "No Karate runner class", "a module without test sources has no runner") +} + +func TestJavaKarateProbeGoesToTheModuleWithTheRunner(t *testing.T) { + root := t.TempDir() + writeJavaSource(t, filepath.Join(root, "api", "src", "test", "java", "example", "ApiTest.java"), "package example;\npublic class ApiTest {}\n") + writeJavaSource(t, filepath.Join(root, "e2e", "src", "test", "java", "example", "E2ETest.java"), "package example;\nimport com.intuit.karate.junit5.Karate;\nclass E2ETest {\n @Karate.Test Karate all() { return Karate.run(\"classpath:example\"); }\n}\n") + projects := []maven.Project{{Directory: "api", Artifact: "api"}, {Directory: "e2e", Artifact: "e2e"}} + module, ok := javaRunnerModule(root, projects, []nativeTest{{File: "api / example.ApiTest"}, {File: "e2e / [example/calc] Calculator"}}, "karate") + require.True(t, ok) + require.Equal(t, "e2e", module) + _, ok = javaRunnerModule(root, projects, []nativeTest{{File: "api / example.ApiTest"}}, "junit5") + require.False(t, ok, "a class probe goes to any module that ran tests") +} + +func TestWithoutSuiteLeavesTheHarnessOut(t *testing.T) { + facts := intake.Facts{TestEventCount: 3, + Tests: []intake.Test{ + {Suite: "example.KarateTest", Name: "all", Attempts: []intake.TestRun{{Status: "pass"}}}, + {Suite: "[ddtest/DdtestValidation] " + javaProbeClass, Name: "ddtestProbe", Attempts: []intake.TestRun{{Status: "skip"}, {Status: "skip"}}}, + }, + Events: []intake.Event{{Type: "test", Tags: map[string]string{"test.suite": "example.KarateTest"}}, {Type: "test", Tags: map[string]string{"test.suite": "[ddtest/DdtestValidation] " + javaProbeClass}}}, + } + left := withoutSuite(facts, "example.KarateTest") + require.Equal(t, 2, left.TestEventCount, "skipping counts the probe's events against every test event") + require.Len(t, left.Tests, 1) + require.Len(t, left.Events, 1) + require.Equal(t, "ddtestProbe", left.Tests[0].Name) + require.Len(t, facts.Tests, 2, "the run's own facts are not changed") +} diff --git a/internal/testdrive/java_language.go b/internal/testdrive/java_language.go index 4530cf15..f76a2ad4 100644 --- a/internal/testdrive/java_language.go +++ b/internal/testdrive/java_language.go @@ -224,7 +224,7 @@ func (javaLanguage) noResultsFinding(reportRuntime, bool, bool, *reportCard, *in func (javaLanguage) pairedFrameworks() map[string]pairedFramework { adapter := pairedFramework{ run: func(t *Testdrive, ctx context.Context, output io.Writer, session *workspace.Session, agent string, spec runSpec) (validationRun, error) { - return t.runJava(ctx, output, session, agent, spec, nil) + return t.runJava(ctx, output, session, agent, spec, javaProbe{}) }, features: func(t *Testdrive, ctx context.Context, output io.Writer, session *workspace.Session, agent string, baseline validationRun, result *validationResult) error { return t.runJavaFeatures(ctx, output, session, agent, baseline, result) @@ -416,4 +416,14 @@ func init() { for name := range (javaLanguage{}).pairedFrameworks() { frameworkFeaturePolicies[name] = frameworkPolicy{SourceCoverage: true, ProbeBodyCounts: true, TestLevelSkipping: true, Paired: true, CheckOnly: true, Preflight: true, Configurations: true, Features: slices.Clone(features)} } + for i, feature := range frameworkFeaturePolicies["munit"].Features { + if feature.Name == "skipping" || feature.Name == "disabled" { + frameworkFeaturePolicies["munit"].Features[i] = featurePolicy{Name: feature.Name, Status: "not supported", Reason: munitSkippingReason} + } + } } + +// munitSkippingReason is why MUnit tests are neither skipped nor disabled: +// dd-java-agent skips a JUnit 4 test as a ParentRunner starts it, and MUnit's +// runner is not one. +const munitSkippingReason = "dd-java-agent cannot skip or disable a Scala MUnit test: it skips JUnit 4 tests through ParentRunner, which MUnit's runner does not extend. This feature is not exercised and is not counted as a passed check." diff --git a/internal/testdrive/java_validation.go b/internal/testdrive/java_validation.go index f7d82dc2..62eba68e 100644 --- a/internal/testdrive/java_validation.go +++ b/internal/testdrive/java_validation.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io" + "io/fs" "maps" "os" "path/filepath" @@ -158,8 +159,8 @@ func readMavenConfig(path string) ([]string, error) { // runJava runs the selected Maven command once in the disposable copy, with // fresh build output, and reads its native Surefire/Failsafe reports. A probe -// run adds probeArgs, which select only the probe. -func (t *Testdrive) runJava(ctx context.Context, output io.Writer, session *workspace.Session, agent string, spec runSpec, probeArgs []string) (run validationRun, runErr error) { +// run adds the probe's arguments, which select only the probe. +func (t *Testdrive) runJava(ctx context.Context, output io.Writer, session *workspace.Session, agent string, spec runSpec, probe javaProbe) (run validationRun, runErr error) { run.Name, run.Role, run.Instrumented, run.ProbeMode = spec.Name, spec.Role, spec.Instrumented, spec.ProbeMode if spec.ProbePath != "" { run.Role = roleProbeControl @@ -234,7 +235,7 @@ func (t *Testdrive) runJava(ctx context.Context, output io.Writer, session *work if spec.ProbePath != "" { // The probe only needs Surefire: checks after the test phase, such as // license headers or coverage gates, would judge the probe file itself. - args = append(maven.UpToTest(args), probeArgs...) + args = append(maven.UpToTest(args), probe.args...) env["DDTEST_PROBE_MODE"] = spec.ProbeMode env["DDTEST_PROBE_STATE"] = filepath.Join(directory, "probe-state") } @@ -268,6 +269,11 @@ func (t *Testdrive) runJava(ctx context.Context, output io.Writer, session *work case spec.ProbePath == "" && maven.IncompleteBuild(run.ExitCode, projects, tests, string(data)): run.ResultError = javaIncompleteBuild } + if probe.harness != "" { + // The runner that starts the probe feature is not a test of its own. + tests = slices.DeleteFunc(tests, func(test maven.Test) bool { return test.Suite == probe.harness }) + run.Facts = withoutSuite(run.Facts, probe.harness) + } run.Tests = javaNativeTests(projects, tests, spec.ProbePath != "") if spec.Instrumented && spec.ProbePath == "" { run.Coverage = javaSourceCoverage(t.checkoutRoot(), run.Facts, run.Tests) @@ -285,6 +291,19 @@ func (t *Testdrive) runJava(ctx context.Context, output io.Writer, session *work return run, nil } +// withoutSuite leaves a suite's tests and their events out of facts. +func withoutSuite(facts intake.Facts, suite string) intake.Facts { + facts.Tests = slices.DeleteFunc(slices.Clone(facts.Tests), func(test intake.Test) bool { + if test.Suite == suite { + facts.TestEventCount -= len(test.Attempts) + return true + } + return false + }) + facts.Events = slices.DeleteFunc(slices.Clone(facts.Events), func(event intake.Event) bool { return event.Tags["test.suite"] == suite }) + return facts +} + // javaModuleLabels maps each project directory to the name dd-java-agent // reports as test.module: the // project's , or its artifactId. A name several projects share, or one @@ -312,7 +331,8 @@ func javaNativeTests(projects []maven.Project, tests []maven.Test, probe bool) [ native := make([]nativeTest, 0, len(tests)) for _, test := range tests { item := nativeTest{File: labels[test.Module] + " / " + test.Suite, Name: test.Name, Status: test.Status, Failure: withoutObjectAddresses(test.Failure)} - if probe && test.Suite == javaProbeClass { + // A Karate suite is the probe feature's path and name. + if probe && (test.Suite == javaProbeClass || strings.HasSuffix(test.Suite, "] "+javaProbeClass)) { name := javaTestName(test.Name) item.probe, item.telemetryName = true, &name } @@ -372,6 +392,10 @@ var javaPluginSuffix = regexp.MustCompile(` (maven-surefire-plugin|maven-failsaf // Surefire reports the name alone. var javaFeatureURI = regexp.MustCompile(`^(?:classpath|file):.*?\.feature:`) +// javaScalaTestRunner is the module dd-java-agent reports for suites the +// scalatest-maven-plugin runs. +const javaScalaTestRunner = "org.scalatest.tools.Runner" + // verifyJavaTelemetry matches every native result to one reported test // execution by module, class, method and status. Equal totals are not enough. func verifyJavaTelemetry(run validationRun) verdict { @@ -387,9 +411,24 @@ func verifyJavaTelemetry(run validationRun) verdict { // written when the agent reported it so, else without its JUnit signature // or TestNG arguments, else also without the invocation index, which a // custom @ParameterizedTest name keeps out of the agent's telemetry. + // The scalatest-maven-plugin's suites are reported under its runner, so + // they are attributed to the one module whose native results hold the suite. + suiteModules := map[string][]string{} + for _, test := range run.Tests { + module, suite, _ := strings.Cut(test.File, " / ") + if !slices.Contains(suiteModules[suite], module) { + suiteModules[suite] = append(suiteModules[suite], module) + } + } + telemetryModule := func(test intake.Test) string { + if modules := suiteModules[test.Suite]; test.Module == javaScalaTestRunner && len(modules) == 1 { + return modules[0] + } + return javaPluginSuffix.ReplaceAllString(test.Module, "") + } reported := map[string]bool{} for _, test := range run.Facts.Tests { - reported[fmt.Sprintf("%q %q %q", javaPluginSuffix.ReplaceAllString(test.Module, ""), javaFeatureURI.ReplaceAllString(test.Suite, ""), javaTelemetryName(test))] = true + reported[fmt.Sprintf("%q %q %q", telemetryModule(test), javaFeatureURI.ReplaceAllString(test.Suite, ""), javaTelemetryName(test))] = true } for _, test := range run.Tests { module, suite, _ := strings.Cut(test.File, " / ") @@ -404,7 +443,7 @@ func verifyJavaTelemetry(run validationRun) verdict { expected[fmt.Sprintf("module %q, class %q, test %q, status %q", module, suite, name, statuses[test.Status])]++ } for _, test := range run.Facts.Tests { - module := javaPluginSuffix.ReplaceAllString(test.Module, "") + module := telemetryModule(test) suite := javaFeatureURI.ReplaceAllString(test.Suite, "") name := javaTelemetryName(test) spans := map[uint64]bool{} @@ -486,37 +525,125 @@ class DdtestValidationTest extends spock.lang.Specification { } ` +// scalaProbeSource is javaProbeSource as a Scala suite of the given class, +// which defines test(name) { ... } in both MUnit and ScalaTest. +func scalaProbeSource(suite string) string { + return `package ddtest + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Paths, StandardOpenOption} + +class DdtestValidationTest extends ` + suite + ` { + test("ddtestProbe") { + val state = Paths.get(System.getenv("DDTEST_PROBE_STATE")) + Files.write(state, "run\n".getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND) + val attempts = Files.readAllLines(state).size + val mode = System.getenv("DDTEST_PROBE_MODE") + if (mode == "fail" || (mode == "fail-once" && attempts == 1)) { + throw new AssertionError("ddtest intentional failure") + } + } +} +` +} + // javaProbeFile is one file of a probe, relative to the checkout. type javaProbeFile struct{ path, source string } // javaProbe is a framework's controlled probe in one module: the files to // write, and the Maven arguments that select only it. Every probe reports the -// class javaProbeClass. +// class javaProbeClass. A probe that a runner class starts names it as its +// harness, whose own results are left out. type javaProbe struct { - files []javaProbeFile - args []string + files []javaProbeFile + args []string + harness string } // javaProbeSources are the probe sources of frameworks whose probe is one // test class, relative to a module. var javaProbeSources = map[string]javaProbeFile{ - "junit4": {filepath.Join("src", "test", "java", "ddtest", "DdtestValidationTest.java"), javaProbeSource("org.junit.Test")}, - "junit5": {filepath.Join("src", "test", "java", "ddtest", "DdtestValidationTest.java"), javaProbeSource("org.junit.jupiter.api.Test")}, - "testng": {filepath.Join("src", "test", "java", "ddtest", "DdtestValidationTest.java"), javaProbeSource("org.testng.annotations.Test")}, - "spock": {filepath.Join("src", "test", "groovy", "ddtest", "DdtestValidationTest.groovy"), spockProbeSource}, + "junit4": {filepath.Join("src", "test", "java", "ddtest", "DdtestValidationTest.java"), javaProbeSource("org.junit.Test")}, + "junit5": {filepath.Join("src", "test", "java", "ddtest", "DdtestValidationTest.java"), javaProbeSource("org.junit.jupiter.api.Test")}, + "testng": {filepath.Join("src", "test", "java", "ddtest", "DdtestValidationTest.java"), javaProbeSource("org.testng.annotations.Test")}, + "spock": {filepath.Join("src", "test", "groovy", "ddtest", "DdtestValidationTest.groovy"), spockProbeSource}, + "scalatest": {filepath.Join("src", "test", "scala", "ddtest", "DdtestValidationTest.scala"), scalaProbeSource("org.scalatest.funsuite.AnyFunSuite")}, + "munit": {filepath.Join("src", "test", "scala", "ddtest", "DdtestValidationTest.scala"), scalaProbeSource("munit.FunSuite")}, } -// javaProbeFrameworks are the frameworks with paired validation; Surefire -// runs and reports each of them. -var javaProbeFrameworks = []string{"junit4", "junit5", "testng", "spock", "cucumber-java"} +// javaProbeFrameworks are the frameworks with paired validation; Surefire or +// the scalatest-maven-plugin runs each of them. +var javaProbeFrameworks = []string{"junit4", "junit5", "testng", "spock", "cucumber-java", "karate", "scalatest", "munit"} // javaProbeFor returns framework's probe in module, or why it has none. func javaProbeFor(checkout, module, framework string, args []string) (javaProbe, string, error) { - if framework == "cucumber-java" { + switch framework { + case "cucumber-java": return javaCucumberProbe(checkout, module, args) + case "karate": + return javaKarateProbe(checkout, module) } source := javaProbeSources[framework] - return javaProbe{files: []javaProbeFile{{filepath.Join(module, source.path), source.source}}, args: javaProbeArguments}, "", nil + probe := javaProbe{files: []javaProbeFile{{filepath.Join(module, source.path), source.source}}, args: javaProbeArguments} + if framework == "scalatest" { + // Surefire runs ScalaTest suites through the JUnit runner they + // declare, so the probe declares the module's. + runner, err := javaScalaTestJUnitRunner(filepath.Join(checkout, module)) + if err != nil { + return javaProbe{}, "", err + } + if runner != "" { + probe.files[0].source = strings.Replace(source.source, "\nclass ", "\n@org.junit.runner.RunWith(classOf["+runner+"])\nclass ", 1) + } + // The scalatest-maven-plugin selects suites by its own property; + // suites that Surefire runs through JUnit match -Dtest. Unlike + // suites, wildcardSuites lets the plugin's other modules, which lack + // the probe, run nothing; an empty suites clears the command's own. + probe.args = append(slices.Clone(javaProbeArguments), "-Dsuites=", "-DwildcardSuites="+javaProbeClass) + } + return probe, "", nil +} + +// javaScalaTestJUnitRunners are ScalaTest's JUnit 4 runners, current and +// before ScalaTest 3.1. +var javaScalaTestJUnitRunners = []string{"org.scalatestplus.junit.JUnitRunner", "org.scalatest.junit.JUnitRunner"} + +// javaScalaTestRunWith is a @RunWith of a class named JUnitRunner, however +// imported. +var javaScalaTestRunWith = regexp.MustCompile(`@(?:[\w.]+\.)?RunWith\(\s*classOf\[\s*(?:[\w.]+\.)?JUnitRunner\s*\]\s*\)`) + +// javaScalaTestJUnitRunner is the ScalaTest JUnit runner a module's Scala +// tests declare with @RunWith, if any: the one whose package the source names, +// as a qualified name or in an import. +func javaScalaTestJUnitRunner(module string) (string, error) { + runner := "" + err := filepath.WalkDir(filepath.Join(module, "src", "test", "scala"), func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if runner != "" || entry.IsDir() || !strings.HasSuffix(path, ".scala") || !entry.Type().IsRegular() { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + text := string(data) + if !javaScalaTestRunWith.MatchString(text) { + return nil + } + for _, candidate := range javaScalaTestJUnitRunners { + if strings.Contains(text, strings.TrimSuffix(candidate, ".JUnitRunner")) { + runner = candidate + return nil + } + } + return nil + }) + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } + return runner, err } // runJavaFeatures writes the probe into a module whose tests ran in the @@ -537,12 +664,10 @@ func (t *Testdrive) runJavaFeatures(ctx context.Context, output io.Writer, sessi unavailableFeatures(result, "No module of the selected Maven command ran native tests, so the controlled probe has nowhere to run.") return nil } - if t.framework.Name() == "cucumber-java" { - // The probe needs the module that holds a Cucumber runner, which an - // inherited dependency hides from reactor order. - if owner, found := javaCucumberModule(checkout, projects, baseline.Tests); found { - module = owner - } + // A Cucumber or Karate probe needs the module that holds the runner, + // which an inherited dependency hides from reactor order. + if owner, found := javaRunnerModule(checkout, projects, baseline.Tests, t.framework.Name()); found { + module = owner } probe, reason, err := javaProbeFor(checkout, module, t.framework.Name(), args) if err != nil { @@ -580,7 +705,7 @@ func (t *Testdrive) runJavaFeatures(ctx context.Context, output io.Writer, sessi } probePath := filepath.Join(checkout, probe.files[0].path) return runProbeFeatures(func(name string, scenario intake.Scenario, mode string) (validationRun, error) { - return t.runJava(ctx, output, session, agent, runSpec{Name: name, Instrumented: true, Scenario: scenario, ProbePath: probePath, ProbeMode: mode}, probe.args) + return t.runJava(ctx, output, session, agent, runSpec{Name: name, Instrumented: true, Scenario: scenario, ProbePath: probePath, ProbeMode: mode}, probe) }, result) } @@ -601,6 +726,51 @@ func javaProbeModule(projects []maven.Project, tests []nativeTest, framework str return candidates[max(declared, 0)].Directory, true } +// javaModuleText names a module directory in a message. +func javaModuleText(module string) string { + if module == "." { + return "the root module" + } + return "module " + module +} + +// javaRunners find the runner class of the frameworks whose probe is +// selected through one, in a module directory. +var javaRunners = map[string]func(module string) (string, error){ + "cucumber-java": func(module string) (string, error) { + runner, _, _, _, _, err := javaCucumberRunnerClass(module) + return runner, err + }, + "karate": func(module string) (string, error) { + runner, _, err := javaKarateRunnerClass(module) + return runner, err + }, +} + +// javaRunnerModule picks the first module, in reactor order, whose tests ran +// in the baseline and that holds a runner class of framework. +func javaRunnerModule(checkout string, projects []maven.Project, tests []nativeTest, framework string) (string, bool) { + find, ok := javaRunners[framework] + if !ok { + return "", false + } + labels := javaModuleLabels(projects) + executed := map[string]bool{} + for _, test := range tests { + module, _, _ := strings.Cut(test.File, " / ") + executed[module] = true + } + for _, project := range projects { + if !executed[labels[project.Directory]] { + continue + } + if runner, err := find(filepath.Join(checkout, project.Directory)); err == nil && runner != "" { + return project.Directory, true + } + } + return "", false +} + // checkJavaPreflight inspects the JDK Maven runs on, the reactor the selected // command builds and the CI JDK and agent selections of this configuration, // without running Maven or project code. diff --git a/internal/testdrive/java_validation_integration_test.go b/internal/testdrive/java_validation_integration_test.go index 107fbb7e..cd9544b0 100644 --- a/internal/testdrive/java_validation_integration_test.go +++ b/internal/testdrive/java_validation_integration_test.go @@ -90,6 +90,10 @@ public class StepDefinitions { @io.cucumber.java.en.Then("the result is {int}") public void check(int expected) { if (result != expected) throw new AssertionError("customer expected failure: " + result); } } ` + const scala = `org.scala-langscala-library2.13.15` + const scalaPlugin = `net.alchim31.mavenscala-maven-plugin4.9.2testCompile` + const karate = `io.karatelabskarate-junit51.5.2test` + const karateFeature = "Feature: Calculator\n Scenario: adds\n * match Java.type('example.Calc').add(2, 2) == 4\n\n Scenario: existing failure\n * match Java.type('example.Calc').add(2, 2) == 5\n\n Scenario Outline: sums \n * match Java.type('example.Calc').add(, 1) == \n Examples:\n | a | b |\n | 1 | 2 |\n | 2 | 3 |\n" // Spock specifications compile with GMavenPlus from src/test/groovy. const gmavenplus = `org.codehaus.gmavenplusgmavenplus-plugin4.1.1addTestSourcescompileTests` testdrive := func(root string, args ...string) (string, error) { @@ -101,9 +105,9 @@ public class StepDefinitions { // CI spreads the scenarios over shards (DDTEST_TEST_SHARD); weights are // relative durations. - scenarios := []string{"junit5", "junit4", "testng", "spock", "cucumber junit4 runner", "cucumber java8 lambdas", "cucumber platform suite", + scenarios := []string{"junit5", "junit4", "testng", "spock", "cucumber junit4 runner", "cucumber java8 lambdas", "cucumber platform suite", "karate parallel runner", "karate junit5 runner", "scalatest", "scalatest junit runner", "munit", "no application coverage", "instrumentation-only failure", "incomplete reactor", "publishing goal", "agent workflow"} - weights := map[string]int{"junit5": 20, "agent workflow": 16, "no application coverage": 6, "instrumentation-only failure": 6, "incomplete reactor": 2, "publishing goal": 1} + weights := map[string]int{"junit5": 20, "scalatest": 6, "scalatest junit runner": 2, "munit": 6, "agent workflow": 16, "no application coverage": 6, "instrumentation-only failure": 6, "incomplete reactor": 2, "publishing goal": 1} selected := shard.SelectedIntegrationScenarios(t, scenarios, weights) run := func(name string, test func(t *testing.T)) { t.Helper() @@ -158,6 +162,21 @@ class ExampleTest extends spock.lang.Specification { {"cucumber platform suite", "cucumber-java", cucumberPlatform, "3.2.5", "", map[string]string{ "src/test/java/example/RunCucumberTest.java": "package example;\n@org.junit.platform.suite.api.Suite\n@org.junit.platform.suite.api.IncludeEngines(\"cucumber\")\n@org.junit.platform.suite.api.SelectClasspathResource(\"example\")\n@org.junit.platform.suite.api.ConfigurationParameter(key = io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME, value = \"example\")\npublic class RunCucumberTest {}\n", "src/test/java/example/StepDefinitions.java": cucumberSteps, "src/test/resources/example/calc.feature": cucumberFeature}}, + // Karate's documented runner is a JUnit test that runs features in + // parallel and fails with them, so it is a native failure too. + {"karate parallel runner", "karate", karate, "3.2.5", "", map[string]string{ + "src/test/java/example/KarateTest.java": "package example;\nclass KarateTest {\n @org.junit.jupiter.api.Test void all() {\n com.intuit.karate.Results results = com.intuit.karate.Runner.path(\"classpath:example\").tags(\"~@ignore\").parallel(2);\n org.junit.jupiter.api.Assertions.assertEquals(0, results.getFailCount(), results.getErrorMessages());\n }\n}\n", + "src/test/resources/example/calc.feature": karateFeature}}, + // Surefire reports a @Karate.Test runner's scenarios as dynamic tests. + {"karate junit5 runner", "karate", karate, "3.2.5", "", map[string]string{ + "src/test/java/example/CalcTest.java": "package example;\nclass CalcTest {\n @com.intuit.karate.junit5.Karate.Test com.intuit.karate.junit5.Karate calc() { return com.intuit.karate.junit5.Karate.run(\"classpath:example/calc.feature\"); }\n}\n", + "src/test/resources/example/calc.feature": karateFeature}}, + // The scalatest-maven-plugin runs the suites; Surefire has none to run. + {"scalatest", "scalatest", scala + `org.scalatestscalatest_2.133.2.19test`, "3.5.3", + scalaPlugin + `org.scalatestscalatest-maven-plugin2.2.0.testtest`, + map[string]string{"src/test/scala/example/CalcTest.scala": "package example\n\nclass CalcTest extends org.scalatest.funsuite.AnyFunSuite {\n test(\"adds\") { assert(Calc.add(2, 2) == 4) }\n test(\"existing failure\") { assert(Calc.add(2, 2) == 5, \"customer expected failure\") }\n ignore(\"ignored\") {}\n}\n"}}, + {"munit", "munit", scala + `org.scalametamunit_2.131.0.4test`, "3.5.3", scalaPlugin, + map[string]string{"src/test/scala/example/CalcTest.scala": "package example\n\nclass CalcTest extends munit.FunSuite {\n test(\"adds\") { assertEquals(Calc.add(2, 2), 4) }\n test(\"existing failure\") { assertEquals(Calc.add(2, 2), 5, \"customer expected failure\") }\n test(\"ignored\".ignore) {}\n}\n"}}, } { run(tc.name, func(t *testing.T) { files := map[string]string{"pom.xml": pomWith("fixture", tc.dependency, tc.surefire, tc.plugins), "src/main/java/example/Calc.java": calc} @@ -172,10 +191,16 @@ class ExampleTest extends spock.lang.Specification { require.Equal(t, "compatible", report.Compatibility.Status) require.Len(t, report.Features, 6) for _, feature := range report.Features { - require.Equal(t, "passed", feature.Status, feature.Name) + // dd-java-agent cannot skip an MUnit test. + want := "passed" + if tc.framework == "munit" && (feature.Name == "skipping" || feature.Name == "disabled") { + want = "not supported" + } + require.Equal(t, want, feature.Status, feature.Name) } for _, run := range report.Runs[:2] { - require.Equal(t, 1, run.TestCounts.Failed, "matching existing failures are acceptable") + // Karate's runner test fails with the feature's failure. + require.Equal(t, cmp.Or(map[string]int{"karate parallel runner": 2}[tc.name], 1), run.TestCounts.Failed, "matching existing failures are acceptable") require.NotZero(t, run.ExitCode) } require.NoDirExists(t, filepath.Join(root, "target"), "Maven runs only in disposable copies") @@ -216,6 +241,19 @@ class ExampleTest extends spock.lang.Specification { }) } + // Surefire runs suites that declare ScalaTest's JUnit runner. dd-java-agent + // 1.67.1 and 1.68.0 fail each test of such a suite ("Could not find test + // suite with descriptor"), so the build reports other results with the + // agent: validation must not pass. + run("scalatest junit runner", func(t *testing.T) { + root := project(t, "scalatest", map[string]string{"pom.xml": pomWith("fixture", scala+`org.scalatestscalatest_2.133.2.19testorg.scalatestplusjunit-4-13_2.133.2.19.0test`, "3.5.3", scalaPlugin), "src/main/java/example/Calc.java": calc, + "src/test/scala/example/CalcTest.scala": "package example\n\n@org.junit.runner.RunWith(classOf[org.scalatestplus.junit.JUnitRunner])\nclass CalcTest extends org.scalatest.funsuite.AnyFunSuite {\n test(\"adds\") { assert(Calc.add(2, 2) == 4) }\n test(\"existing failure\") { assert(Calc.add(2, 2) == 5, \"customer expected failure\") }\n ignore(\"ignored\") {}\n}\n"}) + output, err := testdrive(root, "--framework", "scalatest") + require.Error(t, err, output) + require.Contains(t, output, "Compatibility (this invocation): suspected regression") + require.False(t, read(t, root).Success) + }) + run("no application coverage", func(t *testing.T) { root := project(t, "coverage", map[string]string{"pom.xml": pom("fixture", jupiter), "src/test/java/example/ExampleTest.java": "package example;\npublic class ExampleTest { @org.junit.jupiter.api.Test void passes() { org.junit.jupiter.api.Assertions.assertEquals(4, 2 + 2); } }\n"}) output, err := testdrive(root, "--framework", "junit5") diff --git a/internal/testdrive/java_validation_test.go b/internal/testdrive/java_validation_test.go index dd04c01e..a568ea3b 100644 --- a/internal/testdrive/java_validation_test.go +++ b/internal/testdrive/java_validation_test.go @@ -251,7 +251,7 @@ func TestJavaTemporaryAgentPathCannotContainWhitespace(t *testing.T) { func TestJavaRunsMavenOnlyInADisposableCopy(t *testing.T) { drive := &Testdrive{repositoryRoot: t.TempDir(), framework: framework.NewMaven("junit5", nil)} session := workspace.PlanSession() - _, err := drive.runJava(context.Background(), io.Discard, session, "", runSpec{Name: "baseline"}, nil) + _, err := drive.runJava(context.Background(), io.Discard, session, "", runSpec{Name: "baseline"}, javaProbe{}) require.ErrorContains(t, err, "disposable copy") } @@ -279,6 +279,12 @@ func TestJavaNativeTestsUseTheAgentsModuleNames(t *testing.T) { require.True(t, probe.probe) require.Equal(t, "core / "+javaProbeClass, probe.File, "the probe is matched by module, class and name, as other tests") require.Equal(t, "ddtestProbe", *probe.telemetryName) + karate := javaNativeTests(javaFixtureProjects(), []maven.Test{ + {Module: "core", Suite: "[src/test/resources/ddtest/DdtestValidation] " + javaProbeClass, Name: "ddtestProbe", Status: "passed"}, + {Module: "core", Suite: "[example/calc] Calculator", Name: "adds", Status: "passed"}, + }, true) + require.True(t, karate[0].probe, "a Karate probe is its feature's path and name") + require.False(t, karate[1].probe) } func javaTelemetryRun(native []nativeTest, reported ...intake.Test) validationRun { @@ -317,6 +323,15 @@ func TestJavaTelemetryMatchesEveryNativeResult(t *testing.T) { }, check.Differences) } +func TestJavaTelemetryAttributesScalaTestSuitesToTheirModule(t *testing.T) { + native := []nativeTest{{File: "core / example.CalcTest", Name: "adds", Status: "passed"}} + run := javaTelemetryRun(native, reported(javaScalaTestRunner, "example.CalcTest", "adds", "", intake.TestRun{Status: "pass", SpanID: 1})) + require.Equal(t, verdict{}, verifyJavaTelemetry(run), "the plugin's runner stands for the module that ran the suite") + native = append(native, nativeTest{File: "api / example.CalcTest", Name: "adds", Status: "passed"}) + run = javaTelemetryRun(native, reported(javaScalaTestRunner, "example.CalcTest", "adds", "", intake.TestRun{Status: "pass", SpanID: 1}), reported(javaScalaTestRunner, "example.CalcTest", "adds", "", intake.TestRun{Status: "pass", SpanID: 2})) + require.Equal(t, "inconclusive", verifyJavaTelemetry(run).Status, "a suite two modules ran cannot be attributed") +} + func TestJavaTelemetryRejectsUnusableEvidence(t *testing.T) { native := []nativeTest{{File: "core / example.CoreTest", Name: "works", Status: "passed"}} require.Contains(t, verifyJavaTelemetry(javaTelemetryRun(native)).Reason, "sent no test events") @@ -383,6 +398,10 @@ func TestJavaValidatesSurefireFrameworksWithPairedFeatureControls(t *testing.T) require.True(t, supportsValidationFramework(name), name) require.Len(t, featurePolicies(name), 6) require.True(t, frameworkFeaturePolicies[name].TestLevelSkipping, "dd-java-agent skips a test by its suite and name") + for _, feature := range featurePolicies(name) { + unsupported := name == "munit" && (feature.Name == "skipping" || feature.Name == "disabled") + require.Equal(t, unsupported, feature.Status == "not supported", "%s %s", name, feature.Name) + } require.Equal(t, "java", languageForFramework(name).name()) } for name, source := range javaProbeSources { @@ -391,7 +410,31 @@ func TestJavaValidatesSurefireFrameworksWithPairedFeatureControls(t *testing.T) } require.Contains(t, javaProbeSources["testng"].source, "@org.testng.annotations.Test") require.True(t, strings.HasSuffix(javaProbeSources["spock"].path, ".groovy"), "Spock specifications compile from src/test/groovy") - require.False(t, supportsValidationFramework("karate"), "other Maven frameworks keep reporting-only checks") + require.Contains(t, javaProbeSources["munit"].source, "extends munit.FunSuite") + require.True(t, strings.HasSuffix(javaProbeSources["scalatest"].path, ".scala"), "Scala suites compile from src/test/scala") + probe, reason, err := javaProbeFor(t.TempDir(), "core", "scalatest", nil) + require.NoError(t, err) + require.Empty(t, reason) + require.Contains(t, probe.args, "-DwildcardSuites="+javaProbeClass, "the scalatest-maven-plugin selects suites by its own property, which modules without the probe tolerate") + require.Contains(t, probe.args, "-Dsuites=", "the probe replaces the command's own suite selection") + require.Contains(t, probe.args, "-Dtest="+javaProbeClass, "suites Surefire runs through JUnit match -Dtest") + require.NotContains(t, probe.files[0].source, "RunWith", "the scalatest-maven-plugin needs no JUnit runner") + for _, runner := range javaScalaTestJUnitRunners { + pkg := strings.TrimSuffix(runner, ".JUnitRunner") + for _, imports := range []string{"import " + runner, "import " + pkg + "._", "import " + pkg + ".{JUnitRunner, JUnitSuite}"} { + root := t.TempDir() + writeJavaSource(t, filepath.Join(root, "core", "src", "test", "scala", "example", "CalcTest.scala"), "package example\nimport org.junit.runner.RunWith\n"+imports+"\n@RunWith(classOf[JUnitRunner])\nclass CalcTest extends AnyFunSuite\n") + probe, _, err := javaProbeFor(root, "core", "scalatest", nil) + require.NoError(t, err) + require.Contains(t, probe.files[0].source, "\n@org.junit.runner.RunWith(classOf["+runner+"])\nclass DdtestValidationTest extends org.scalatest.funsuite.AnyFunSuite", "Surefire runs the probe through the module's JUnit runner: %s", imports) + } + } + // A JUnit runner of another library is not ScalaTest's. + root := t.TempDir() + writeJavaSource(t, filepath.Join(root, "core", "src", "test", "scala", "example", "CalcTest.scala"), "package example\nimport org.other.JUnitRunner\n@RunWith(classOf[JUnitRunner])\nclass CalcTest\n") + probe, _, err = javaProbeFor(root, "core", "scalatest", nil) + require.NoError(t, err) + require.NotContains(t, probe.files[0].source, "RunWith") require.Equal(t, "java", languageForFramework("testng").name()) require.Equal(t, "JUnit 5", displayName("junit5")) require.Equal(t, "Java", displayName("java")) diff --git a/internal/testdrive/preflight_dispatch_test.go b/internal/testdrive/preflight_dispatch_test.go index ef54eff9..770fa258 100644 --- a/internal/testdrive/preflight_dispatch_test.go +++ b/internal/testdrive/preflight_dispatch_test.go @@ -34,6 +34,9 @@ func TestEveryFrameworkWithPreflightUsesItsOwnChecks(t *testing.T) { "testng": framework.NewMaven("testng", platform.NewJava()), "spock": framework.NewMaven("spock", platform.NewJava()), "cucumber-java": framework.NewMaven("cucumber-java", platform.NewJava()), + "karate": framework.NewMaven("karate", platform.NewJava()), + "scalatest": framework.NewMaven("scalatest", platform.NewJava()), + "munit": framework.NewMaven("munit", platform.NewJava()), } for name, policy := range frameworkFeaturePolicies { if !policy.Preflight { @@ -47,7 +50,7 @@ func TestEveryFrameworkWithPreflightUsesItsOwnChecks(t *testing.T) { switch name { case "rspec", "minitest": require.Contains(t, function, ".checkRubyPreflight", name) - case "junit4", "junit5", "testng", "spock", "cucumber-java": + case "junit4", "junit5", "testng", "spock", "cucumber-java", "karate", "scalatest", "munit": require.Contains(t, function, ".checkJavaPreflight", name) default: require.Contains(t, function, ".check"+displayName(name)+"Preflight", name) diff --git a/skills/ddtest-onboarding-eval/SKILL.md b/skills/ddtest-onboarding-eval/SKILL.md index 0d7678c1..1aab15a6 100644 --- a/skills/ddtest-onboarding-eval/SKILL.md +++ b/skills/ddtest-onboarding-eval/SKILL.md @@ -62,7 +62,7 @@ container-native private `/tmp/bundle` as Bundler's path to avoid bind-mount filesystem overhead; it is removed with the container, while lockfiles and both reports stay in the checkout until collection. Ruby's `TMPDIR` also uses the container's private executable tmpfs; no host cache is mounted there. -For `--framework junit4`, `junit5`, `testng`, `spock` or `cucumber-java`, build `--target java`: it adds a +For `--framework junit4`, `junit5`, `testng`, `spock`, `cucumber-java`, `karate`, `scalatest` or `munit`, build `--target java`: it adds a digest-pinned Temurin JDK 21 (select another with `--build-arg JDK_IMAGE=eclipse-temurin:-jdk@sha256:`) and a checksum-verified Maven. The runner records the JDK's release version before diff --git a/skills/ddtest-onboarding-eval/scripts/run_eval.py b/skills/ddtest-onboarding-eval/scripts/run_eval.py index 2e3401e6..7396703f 100644 --- a/skills/ddtest-onboarding-eval/scripts/run_eval.py +++ b/skills/ddtest-onboarding-eval/scripts/run_eval.py @@ -178,7 +178,7 @@ def unfinished(self): # reactors in disposable copies) repeatedly; they need longer foreground runs # and container-native scratch storage. RUBY_FRAMEWORKS = ("rspec", "minitest") -JAVA_FRAMEWORKS = ("junit4", "junit5", "testng", "spock", "cucumber-java") +JAVA_FRAMEWORKS = ("junit4", "junit5", "testng", "spock", "cucumber-java", "karate", "scalatest", "munit") def agent_environment(agent, timeout, framework=None): diff --git a/skills/ddtest-onboarding-eval/scripts/test_contract.py b/skills/ddtest-onboarding-eval/scripts/test_contract.py index b9a3bdcd..0f502c78 100644 --- a/skills/ddtest-onboarding-eval/scripts/test_contract.py +++ b/skills/ddtest-onboarding-eval/scripts/test_contract.py @@ -19,7 +19,7 @@ def test_actual_go_reports_supply_every_consumed_field(self): check=False,capture_output=True,text=True,timeout=180) self.assertEqual(command.returncode,0,command.stdout+command.stderr) reports=json.loads(output.read_text()) - self.assertEqual(len(reports),28) + self.assertEqual(len(reports),34) for report in reports: with self.subTest(framework=report['framework'],check_only=report['check_only']): self.assertIsInstance(report['schema_version'],int) diff --git a/skills/ddtest-onboarding-eval/scripts/test_run_eval.py b/skills/ddtest-onboarding-eval/scripts/test_run_eval.py index cb1032b2..f7a17941 100644 --- a/skills/ddtest-onboarding-eval/scripts/test_run_eval.py +++ b/skills/ddtest-onboarding-eval/scripts/test_run_eval.py @@ -596,7 +596,7 @@ def test_long_command_output_is_bounded_without_losing_command(self): class FrameworkEnvironmentTests(unittest.TestCase): def test_long_matrix_timeout_is_scoped_and_bounded(self): - for framework in ("cypress", "rspec", "minitest", "junit4", "junit5", "testng", "spock", "cucumber-java"): + for framework in ("cypress", "rspec", "minitest", "junit4", "junit5", "testng", "spock", "cucumber-java", "karate", "scalatest", "munit"): self.assertEqual(run_eval.agent_environment("claude", 7200, framework)["BASH_MAX_TIMEOUT_MS"], "3600000") self.assertEqual(run_eval.agent_environment("claude", 1800, framework)["BASH_MAX_TIMEOUT_MS"], "1800000") self.assertEqual(run_eval.agent_environment("claude", 30, framework)["BASH_MAX_TIMEOUT_MS"], "30000")