Validate Karate, ScalaTest and MUnit with paired feature controls

README.md

@@ -19,7 +19,7 @@ For Go, `ddtest instrument` reports a project's tests to Test Optimization
 directly; see [Go tests with ddtest instrument](#go-tests-with-ddtest-instrument).
 
 `ddtest onboard` and `ddtest testdrive` also support Java projects built with
-Maven; JUnit 4, JUnit 5, TestNG, Spock and Cucumber JVM get paired validation. `ddtest plan` and `ddtest run`
+Maven; JUnit 4, JUnit 5, TestNG, Spock, Cucumber JVM, Karate, ScalaTest and MUnit get paired validation. `ddtest plan` and `ddtest run`
 do not support Java, because the Java agent performs Test Impact Analysis inside
 the Maven build. See [Java validation](docs/onboarding.md#java-validation).
 

docs/onboarding.md

@@ -455,21 +455,32 @@ select the framework the project uses (`junit4`, `junit5`, `testng`, `spock`,
 contains a JavaScript, Python or Ruby project keeps that platform by default;
 select Java with `--platform java` or a Maven `--framework`.
 
-JUnit 4, JUnit 5, TestNG, Spock and Cucumber JVM support plain `ddtest testdrive --all --check-only --yes`
+Every Maven framework supports plain `ddtest testdrive --all --check-only --yes`
 and full `ddtest testdrive --all --yes` validation, or a focused
 `ddtest testdrive --framework junit5 --command './mvnw -B verify' --yes`. Each
 configuration runs in a disposable copy with fresh build output, without and with
-dd-java-agent. Native Surefire/Failsafe results must match (identities, outcomes,
+dd-java-agent. Native results (Surefire/Failsafe reports, the
+scalatest-maven-plugin's `target/scalatest-reports` and Karate's
+`target/karate-reports` feature reports) must match (identities, outcomes,
 failure types and messages, exit code), each must match one reported test
 execution, the reporting run must cover repository application sources, and all
 six feature controls run with a probe test in a module of the selected framework
-(a Groovy specification for Spock, compiled by the project's GMavenPlus setup).
+(a Groovy specification for Spock, compiled by the project's GMavenPlus setup,
+and a Scala suite for ScalaTest and MUnit, compiled by its scala-maven-plugin).
 For Cucumber JVM the probe is a feature with a step in the glue package of the
 module's runner class (`@RunWith(Cucumber.class)` or a JUnit Platform suite of the
 `cucumber` engine), so the project's own glue, hooks and object factory load;
 the runner is selected with the probe feature in place of the project's. A
 module without such a runner, or a suite that pins its own features or tags,
-leaves the features inconclusive. Probe builds stop at the `test` phase, so
+leaves the features inconclusive. For Karate the probe is a feature that the
+module's runner class (`@Karate.Test`, `@RunWith(Karate.class)` or a JUnit test
+calling `Runner.path`) runs through `karate.options`, so `karate-config.js` and
+`karate.env` still apply; the runner's own JUnit result is left out. MUnit's
+runner is not one dd-java-agent can skip tests in, so its skipping and disabled
+features are reported as not supported. A ScalaTest suite that Surefire runs
+through ScalaTest's `JUnitRunner` gets a probe with the same runner; dd-java-agent
+1.67.1 and 1.68.0 fail every test of such a suite, which validation reports as a
+suspected regression. Probe builds stop at the `test` phase, so
 checks after the tests, such as license headers, do not judge the probe.
 Matching pre-existing failures are permitted; a build that stops for anything
 other than native test failures stays inconclusive.
@@ -495,7 +506,6 @@ major is validated on the local JDK as a substitute, recorded in
 a decision for the user, and is then reported as a caveat. Validate that JDK
 instead by running testdrive with `JAVA_HOME` set to it. `ddtest plan` and `ddtest run` do
 not support Java: the agent performs Test Impact Analysis inside the Maven build.
-Karate, ScalaTest and MUnit run reporting-only checks.
 
 Coding agents start with `ddtest onboard --agent`, as for Ruby: there is no
 automatic Maven CI edit. Until every discovered test step of the framework runs after a

internal/maven/reports.go

@@ -7,6 +7,7 @@ package maven
 
 import (
 	"cmp"
+	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"fmt"
@@ -17,7 +18,8 @@ import (
 	"strings"
 )
 
-// Test is one test case from a module's native Surefire or Failsafe report.
+// Test is one test case from a module's native Surefire, Failsafe, ScalaTest
+// or Karate report.
 type Test struct {
 	// Module is the project directory relative to the inspected root.
 	Module, Suite, Name, Status string
@@ -30,7 +32,7 @@ type Test struct {
 var ErrNoTestCases = errors.New("native reports contain no test cases")
 
 // ErrNoReports reports a build that wrote no native report for any module.
-var ErrNoReports = errors.New("no native TEST-*.xml reports under the modules' target/surefire-reports or target/failsafe-reports; configure Surefire or Failsafe XML reporting and run the intended Maven profile")
+var ErrNoReports = errors.New("no native reports under the modules' target/surefire-reports, target/failsafe-reports, target/scalatest-reports or target/karate-reports; configure Surefire or Failsafe XML reporting (the scalatest-maven-plugin writes it with <junitxml>.</junitxml>) and run the intended Maven profile")
 
 type reportCase struct {
 	Class    string          `xml:"classname,attr"`
@@ -51,37 +53,62 @@ type reportSuite struct {
 	Cases []reportCase `xml:"testcase"`
 }
 
-// reportDirectories are where Surefire and Failsafe write one TEST-*.xml per class.
-var reportDirectories = []string{filepath.Join("target", "surefire-reports"), filepath.Join("target", "failsafe-reports")}
+// reportDirectories are where Surefire, Failsafe and the scalatest-maven-plugin
+// write one TEST-*.xml per class.
+var reportDirectories = []string{filepath.Join("target", "surefire-reports"), filepath.Join("target", "failsafe-reports"), filepath.Join("target", "scalatest-reports")}
+
+// karateReports is where Karate writes one *.karate-json.txt per feature.
+var karateReports = filepath.Join("target", "karate-reports")
 
 // ReadReports reads the native reports of every project under root. Call it
 // only on fresh build output: reports from an earlier build are read as well.
 func ReadReports(root string, projects []Project) ([]Test, error) {
 	var tests []Test
 	reports := 0
 	for _, project := range projects {
+		var paths []string
 		for _, directory := range reportDirectories {
-			paths, err := filepath.Glob(filepath.Join(root, project.Directory, directory, "TEST-*.xml"))
+			found, err := filepath.Glob(filepath.Join(root, project.Directory, directory, "TEST-*.xml"))
 			if err != nil {
 				return nil, err
 			}
-			slices.Sort(paths)
-			for _, path := range paths {
-				info, err := os.Lstat(path)
-				if err != nil {
-					return nil, err
-				}
-				if !info.Mode().IsRegular() {
-					return nil, fmt.Errorf("native test report is not a regular file: %s", path)
-				}
-				read, err := readReport(path, project.Directory)
-				if err != nil {
-					return nil, err
-				}
-				reports++
-				tests = append(tests, read...)
+			slices.Sort(found)
+			paths = append(paths, found...)
+		}
+		features, err := filepath.Glob(filepath.Join(root, project.Directory, karateReports, "*.karate-json.txt"))
+		if err != nil {
+			return nil, err
+		}
+		slices.Sort(features)
+		var module, karate []Test
+		var karatePaths []string
+		for _, path := range append(paths, features...) {
+			info, err := os.Lstat(path)
+			if err != nil {
+				return nil, err
+			}
+			if !info.Mode().IsRegular() {
+				return nil, fmt.Errorf("native test report is not a regular file: %s", path)
 			}
+			var read []Test
+			if strings.HasSuffix(path, ".karate-json.txt") {
+				var feature string
+				read, feature, err = readKarateReport(path, project.Directory)
+				karate, karatePaths = append(karate, read...), append(karatePaths, feature)
+			} else {
+				read, err = readReport(path, project.Directory)
+				module = append(module, read...)
+			}
+			if err != nil {
+				return nil, err
+			}
+			reports++
 		}
+		// Surefire reports each Karate scenario of a JUnit 5 runner under its
+		// feature path, as example/calc, and may lose some; the feature's
+		// own report replaces them.
+		module = slices.DeleteFunc(module, func(test Test) bool { return slices.Contains(karatePaths, test.Suite) })
+		tests = append(append(tests, module...), karate...)
 	}
 	switch {
 	case reports == 0:
@@ -92,6 +119,59 @@ func ReadReports(root string, projects []Project) ([]Test, error) {
 	return tests, nil
 }
 
+// karateFeature is the part of a Karate feature's JSON report that names its
+// scenarios and their outcomes.
+type karateFeature struct {
+	Name         string `json:"name"`
+	RelativePath string `json:"relativePath"`
+	CallDepth    int    `json:"callDepth"`
+	Scenarios    []struct {
+		Name   string            `json:"name"`
+		RefID  string            `json:"refId"`
+		Failed bool              `json:"failed"`
+		Error  string            `json:"error"`
+		Steps  []json.RawMessage `json:"stepResults"`
+	} `json:"scenarioResults"`
+}
+
+// readKarateReport reads a feature's Karate JSON report. Each scenario is a
+// test of the suite dd-java-agent reports, "[<feature>] <name>", where feature
+// is the feature's path without .feature, which readKarateReport also returns.
+// As in dd-java-agent, an unnamed scenario is named by its reference, as
+// [2:6], and one that ran no step, as a hook skips it, is skipped.
+func readKarateReport(path, module string) ([]Test, string, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, "", err
+	}
+	var report karateFeature
+	if err := json.Unmarshal(data, &report); err != nil {
+		return nil, "", fmt.Errorf("parse native report %s: %w", path, err)
+	}
+	if report.RelativePath == "" {
+		return nil, "", fmt.Errorf("native report %s names no feature path", path)
+	}
+	feature := strings.TrimSuffix(report.RelativePath, ".feature")
+	// A feature another one calls is part of the caller's scenario.
+	if report.CallDepth > 0 {
+		return nil, feature, nil
+	}
+	var tests []Test
+	for _, scenario := range report.Scenarios {
+		test := Test{Module: module, Suite: "[" + feature + "] " + report.Name, Name: cmp.Or(strings.TrimSpace(scenario.Name), scenario.RefID), Status: "passed"}
+		if len(scenario.Steps) == 0 {
+			test.Status = "skipped"
+		}
+		if scenario.Failed {
+			// A match failure's first line names only the operator; the
+			// values that differ follow on the next lines.
+			test.Status, test.Failure = "failed", "karate: "+strings.Join(strings.Fields(scenario.Error), " ")+"\n"
+		}
+		tests = append(tests, test)
+	}
+	return tests, feature, nil
+}
+
 func readReport(path, module string) ([]Test, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
@@ -146,10 +226,10 @@ func readReport(path, module string) ([]Test, error) {
 	return tests, nil
 }
 
-var failedGoal = regexp.MustCompile(`^\[ERROR\] Failed to execute goal ([^ ]+) \([^)]*\) on project ([^:]+): (.*)$`)
+var failedGoal = regexp.MustCompile(`^\[ERROR\] Failed to execute goal ([^ ]+) \([^)]*\) on project ([^:]+):(.*)$`)
 
 // testGoals are the plugin goals whose failure a native test failure explains.
-var testGoals = []string{"org.apache.maven.plugins:maven-surefire-plugin:test", "org.apache.maven.plugins:maven-failsafe-plugin:verify"}
+var testGoals = []string{"org.apache.maven.plugins:maven-surefire-plugin:test", "org.apache.maven.plugins:maven-failsafe-plugin:verify", "org.scalatest:scalatest-maven-plugin:test"}
 
 var ansiEscape = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`)
 
@@ -167,13 +247,14 @@ func IncompleteBuild(exitCode int, projects []Project, tests []Test, output stri
 		failedModules[test.Module] = failedModules[test.Module] || test.Status == "failed"
 	}
 	goals := 0
-	for _, line := range strings.Split(ansiEscape.ReplaceAllString(output, ""), "\n") {
+	lines := strings.Split(ansiEscape.ReplaceAllString(output, ""), "\n")
+	for i, line := range lines {
 		line = strings.TrimSpace(line)
 		if !strings.HasPrefix(line, "[ERROR] Failed to execute goal") {
 			continue
 		}
 		match := failedGoal.FindStringSubmatch(line)
-		if match == nil || !strings.HasPrefix(match[3], "There are test failures") {
+		if match == nil || !testFailures(strings.TrimSpace(match[3]), lines[i+1:]) {
 			return true
 		}
 		// The goal is group:artifact:version:goal; the version varies.
@@ -190,3 +271,22 @@ func IncompleteBuild(exitCode int, projects []Project, tests []Test, output stri
 	}
 	return exitCode != 0 && goals == 0
 }
+
+// testFailures reports whether a failed goal's message, and the error lines
+// after it, describe native test failures. Surefire 3.5 states no message when
+// every failed test is an error, and points at the reports on the next lines.
+func testFailures(message string, next []string) bool {
+	if message != "" {
+		return strings.HasPrefix(message, "There are test failures")
+	}
+	for _, line := range next {
+		detail, ok := strings.CutPrefix(strings.TrimSpace(line), "[ERROR]")
+		if !ok {
+			return false
+		}
+		if detail = strings.TrimSpace(detail); detail != "" {
+			return strings.HasPrefix(detail, "See ") && strings.HasSuffix(detail, " for the individual test results.")
+		}
+	}
+	return false
+}

internal/maven/reports_test.go

@@ -38,11 +38,49 @@ func TestReadReportsReadsSurefireAndFailsafeResults(t *testing.T) {
 	}, tests)
 }
 
+func TestReadReportsReadsScalaTestAndKarateResults(t *testing.T) {
+	root := t.TempDir()
+	projects := []Project{{Directory: "scala"}, {Directory: "api"}}
+	writePOM(t, root, "scala/target/scalatest-reports/TEST-example.CalcTest.xml", `<testsuite name="example.CalcTest"><testcase classname="example.CalcTest" name="adds"/></testsuite>`)
+	// The plugin's discovery wrapper holds no test case.
+	writePOM(t, root, "scala/target/scalatest-reports/TEST-org.scalatest.tools.DiscoverySuite-1.xml", `<testsuite name="org.scalatest.tools.DiscoverySuite" tests="0"/>`)
+	// Surefire reports a JUnit 5 runner's scenarios by feature path, and here
+	// loses the failing one; the feature's own report replaces them.
+	writePOM(t, root, "api/target/surefire-reports/TEST-example.CalcTest.xml", `<testsuite name="example.CalcTest">
+  <testcase classname="example/calc" name="[1:2] adds"/>
+  <testcase classname="example.UnitTest" name="checks"/>
+</testsuite>`)
+	writePOM(t, root, "api/target/karate-reports/example.calc.karate-json.txt", `{"name": "Calculator", "relativePath": "example/calc.feature", "callDepth": 0, "scenarioResults": [
+  {"name": "adds", "refId": "[1:2]", "failed": false, "stepResults": [{}]},
+  {"name": "existing failure", "refId": "[2:6]", "failed": true, "error": "match failed: EQUALS\n  4\n  5", "stepResults": [{}]},
+  {"name": "", "refId": "[3:10]", "failed": false, "stepResults": [{}]},
+  {"name": "skipped by a hook", "refId": "[4:14]", "failed": false, "stepResults": []}
+]}`)
+	// A called feature is part of its caller's scenario.
+	writePOM(t, root, "api/target/karate-reports/example.helper.karate-json.txt", `{"name": "Helper", "relativePath": "example/helper.feature", "callDepth": 1, "scenarioResults": [{"name": "login"}]}`)
+
+	tests, err := ReadReports(root, projects)
+	require.NoError(t, err)
+	require.Equal(t, []Test{
+		{Module: "scala", Suite: "example.CalcTest", Name: "adds", Status: "passed"},
+		{Module: "api", Suite: "example.UnitTest", Name: "checks", Status: "passed"},
+		{Module: "api", Suite: "[example/calc] Calculator", Name: "adds", Status: "passed"},
+		{Module: "api", Suite: "[example/calc] Calculator", Name: "existing failure", Status: "failed", Failure: "karate: match failed: EQUALS 4 5\n"},
+		{Module: "api", Suite: "[example/calc] Calculator", Name: "[3:10]", Status: "passed"},
+		{Module: "api", Suite: "[example/calc] Calculator", Name: "skipped by a hook", Status: "skipped"},
+	}, tests)
+
+	writePOM(t, root, "api/target/karate-reports/example.other.karate-json.txt", `{"name": "Other"}`)
+	_, err = ReadReports(root, projects)
+	require.ErrorContains(t, err, "names no feature path")
+}
+
 func TestReadReportsDistinguishesMissingAndEmptyResults(t *testing.T) {
 	root := t.TempDir()
 	projects := []Project{{Directory: "."}}
 	_, err := ReadReports(root, projects)
 	require.ErrorIs(t, err, ErrNoReports)
+	require.ErrorContains(t, err, "<junitxml>.</junitxml>", "the scalatest-maven-plugin writes no XML by default")
 
 	writePOM(t, root, "target/surefire-reports/TEST-example.EmptyTest.xml", `<testsuite name="example.EmptyTest"/>`)
 	_, err = ReadReports(root, projects)
@@ -85,6 +123,9 @@ func TestIncompleteBuild(t *testing.T) {
 		{"successful build", 0, []Test{{Module: "core", Status: "passed"}}, "", false},
 		{"native test failure", 1, failedCore, testFailure, false},
 		{"ANSI-colored test failure", 1, failedCore, "\x1b[1;31m" + testFailure + "\x1b[m", false},
+		{"ScalaTest plugin failure", 1, failedCore, "[ERROR] Failed to execute goal org.scalatest:scalatest-maven-plugin:2.2.0:test (test) on project core: There are test failures -> [Help 1]\n", false},
+		{"Surefire 3.5 test errors only", 1, failedCore, "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:3.5.3:test (default-test) on project core: \n[ERROR] \n[ERROR] See /work/core/target/surefire-reports for the individual test results.\n", false},
+		{"failure without a message or reports", 1, failedCore, "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:3.5.3:test (default-test) on project core: \n[ERROR] \n[ERROR] The forked VM terminated without properly saying goodbye.\n", true},
 		{"Failsafe verification failure", 1, failedCore, "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-failsafe-plugin:3.5.3:verify (default) on project core: There are test failures.\n", false},
 		{"compilation failure", 1, nil, "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.13.0:compile (default-compile) on project api: Compilation failure\n", true},
 		{"later module fails to compile", 1, failedCore, testFailure + "[ERROR] Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.13.0:compile (default-compile) on project api: Compilation failure\n", true},

internal/testdrive/java_cucumber.go

@@ -112,7 +112,7 @@ func javaCucumberProbe(checkout, module string, args []string) (javaProbe, strin
 		return javaProbe{}, "", err
 	}
 	if runner == "" {
-		return javaProbe{}, "No Cucumber runner class (@RunWith(Cucumber.class) or a JUnit Platform suite of the cucumber engine) was found in " + module + "'s test sources, so the probe feature cannot be selected alone.", nil
+		return javaProbe{}, "No Cucumber runner class (@RunWith(Cucumber.class) or a JUnit Platform suite of the cucumber engine) was found in the test sources of " + javaModuleText(module) + ", so the probe feature cannot be selected alone.", nil
 	}
 	// The selected command's own glue property wins over the runner's.
 	for i := 0; i < len(args); i++ {
@@ -189,26 +189,6 @@ func javaCucumberRunnerClass(module string) (runner, glue string, suite, pinned
 	return runner, javaGluePackage(glue), suite, pinned, lambda, err
 }
 
-// javaCucumberModule picks the first module, in reactor order, whose tests ran
-// in the baseline and that holds a Cucumber runner class.
-func javaCucumberModule(checkout string, projects []maven.Project, tests []nativeTest) (string, bool) {
-	labels := javaModuleLabels(projects)
-	executed := map[string]bool{}
-	for _, test := range tests {
-		module, _, _ := strings.Cut(test.File, " / ")
-		executed[module] = true
-	}
-	for _, project := range projects {
-		if !executed[labels[project.Directory]] {
-			continue
-		}
-		if runner, _, _, _, _, err := javaCucumberRunnerClass(filepath.Join(checkout, project.Directory)); err == nil && runner != "" {
-			return project.Directory, true
-		}
-	}
-	return "", false
-}
-
 // javaGluePackage is the first package of a glue list; a path such as
 // classpath:com/example names the package com.example.
 func javaGluePackage(glue string) string {