From 3f5bbf64ee6c0110b45080468624cee68dc61c83 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Mon, 16 Nov 2026 16:35:17 +0100 Subject: [PATCH] Lint: rule packs can declare overrides; SARIF rule descriptions no longer come from transform names, addresses #9500 --- .../org/apache/hop/lint/LintReportWriter.java | 15 ++++++- .../org/apache/hop/lint/LintRuleDetails.java | 20 ++++++--- .../lint/registry/AbstractYamlRulePack.java | 13 ++++++ .../hop/lint/registry/YamlRulePackParser.java | 33 ++++++++++++++ .../apache/hop/lint/LintReportWriterTest.java | 44 +++++++++++++++++++ .../hop/lint/registry/RuleRegistryTest.java | 34 ++++++++++++++ .../hop/lint/registry/test-override-pack.yml | 36 +++++++++++++++ 7 files changed, 187 insertions(+), 8 deletions(-) create mode 100644 plugins/misc/lint/src/test/resources/org/apache/hop/lint/registry/test-override-pack.yml diff --git a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintReportWriter.java b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintReportWriter.java index cfbf4bcc084..25b4a53f8d7 100644 --- a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintReportWriter.java +++ b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintReportWriter.java @@ -168,9 +168,8 @@ private static String renderSarif( // one rule id while carrying the transform name as their result name, so using that // here would label the rule after whichever transform happened to be reported first. rule.put("name", id); - rule.putObject("shortDescription") - .put("text", result.getRuleName() != null ? result.getRuleName() : id); LintRuleDetails details = result.getRuleDetails(); + rule.putObject("shortDescription").put("text", shortDescription(id, result, details)); if (!details.description().isBlank()) { rule.putObject("fullDescription").put("text", details.description()); } @@ -235,6 +234,18 @@ private static void putTags(ObjectNode target, LintRuleDetails details) { }); } + /** + * A remark from Hop's own checks carries the transform or action it was reported on as its rule + * name, so the first one reported would describe the rule for every finding of it. Those take the + * name of the native rule that classified them, or the rule id. + */ + private static String shortDescription(String id, LintResult result, LintRuleDetails details) { + if (result.getOrigin() == LintResult.Origin.HOP_NATIVE) { + return details.name().isBlank() ? id : details.name(); + } + return result.getRuleName() != null ? result.getRuleName() : id; + } + private static String sarifLevel(String severity) { if ("ERROR".equalsIgnoreCase(severity)) { return "error"; diff --git a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintRuleDetails.java b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintRuleDetails.java index c7b05f9c678..17ecd8a607f 100644 --- a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintRuleDetails.java +++ b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintRuleDetails.java @@ -27,27 +27,35 @@ * What a finding carries about the rule that produced it, for the reports to write out. * *

None of it changes what the rule checks: it is the rule's description, where to read more - * about it, and the tags its pack gave it. + * about it, and the tags its pack gave it, with its name for the reports that describe the rule. * + * @param name the rule's name, empty when it has none * @param description the rule's description, empty when it has none * @param helpUri where to read more about the rule, or null * @param tags the rule's tags, each key holding one or more values */ -public record LintRuleDetails(String description, String helpUri, Map> tags) { +public record LintRuleDetails( + String name, String description, String helpUri, Map> tags) { - public static final LintRuleDetails NONE = new LintRuleDetails("", null, Map.of()); + public static final LintRuleDetails NONE = new LintRuleDetails("", "", null, Map.of()); public LintRuleDetails { + name = name != null ? name : ""; description = description != null ? description : ""; tags = copyOf(tags); } + public LintRuleDetails(String description, String helpUri, Map> tags) { + this("", description, helpUri, tags); + } + /** The details of a rule, or {@link #NONE} when there is no rule. */ public static LintRuleDetails of(CustomLintRule rule) { if (rule == null) { return NONE; } - return new LintRuleDetails(rule.getDescription(), rule.getHelpUri(), rule.getTags()); + return new LintRuleDetails( + rule.getName(), rule.getDescription(), rule.getHelpUri(), rule.getTags()); } /** @@ -55,10 +63,10 @@ public static LintRuleDetails of(CustomLintRule rule) { * *

A remark from Hop's own checks with an error code of its own is reported under that code. * The tags of the native rule that classified it apply to it, but that rule's description and - * help link are about the rule, not about the code. + * help link are about the rule, not about the code; so is its name. */ public LintRuleDetails tagsOnly() { - return hasTags() ? new LintRuleDetails("", null, tags) : NONE; + return hasTags() ? new LintRuleDetails("", "", null, tags) : NONE; } public boolean hasTags() { diff --git a/plugins/misc/lint/src/main/java/org/apache/hop/lint/registry/AbstractYamlRulePack.java b/plugins/misc/lint/src/main/java/org/apache/hop/lint/registry/AbstractYamlRulePack.java index 06875e8f08f..bd134b35a48 100644 --- a/plugins/misc/lint/src/main/java/org/apache/hop/lint/registry/AbstractYamlRulePack.java +++ b/plugins/misc/lint/src/main/java/org/apache/hop/lint/registry/AbstractYamlRulePack.java @@ -27,6 +27,7 @@ public abstract class AbstractYamlRulePack implements IHopLintRulePack { private final RulePackOwner owner; private final int priority; private final String resourcePath; + private List overrides; protected AbstractYamlRulePack( String packId, String displayName, RulePackOwner owner, int priority, String resourcePath) { @@ -57,6 +58,18 @@ public int getPriority() { return priority; } + /** The rules this pack replaces, as declared in the {@code overrides:} of its YAML pack block. */ + @Override + public synchronized List getOverrides() { + if (overrides == null) { + overrides = + List.copyOf( + YamlRulePackParser.readOverridesWithAdjacentFallback( + resourcePath, packId, owner, getClass())); + } + return overrides; + } + @Override public List loadRules() { return YamlRulePackParser.loadRulesWithAdjacentFallback( diff --git a/plugins/misc/lint/src/main/java/org/apache/hop/lint/registry/YamlRulePackParser.java b/plugins/misc/lint/src/main/java/org/apache/hop/lint/registry/YamlRulePackParser.java index 9d1c6620e05..76613d69ced 100644 --- a/plugins/misc/lint/src/main/java/org/apache/hop/lint/registry/YamlRulePackParser.java +++ b/plugins/misc/lint/src/main/java/org/apache/hop/lint/registry/YamlRulePackParser.java @@ -293,6 +293,39 @@ public static List loadRulesWithAdjacentFallback( } } + /** + * The {@code pack.overrides:} list of a pack shipped as a classpath resource, falling back to the + * YAML beside the plugin jar the same way {@link #loadRulesWithAdjacentFallback} does. A pack + * that cannot be read declares no overrides; loading its rules reports the error. + */ + public static List readOverridesWithAdjacentFallback( + String resourcePath, String packId, RulePackOwner owner, Class anchor) { + if (Utils.isEmpty(resourcePath)) { + return Collections.emptyList(); + } + try { + for (ClassLoader loader : classLoadersToTry(anchor)) { + if (loader == null) { + continue; + } + try (InputStream inputStream = loader.getResourceAsStream(resourcePath)) { + if (inputStream != null) { + return parsePackMetadata(readYaml(inputStream, resourcePath), packId, owner, 0) + .overrides(); + } + } + } + File adjacent = findPackYamlAdjacentTo(anchor, resourcePath); + if (adjacent != null) { + return readPackMetadata(adjacent, packId, owner, 0).overrides(); + } + } catch (IOException | RuntimeException e) { + LogChannel.GENERAL.logError( + "Failed to read the overrides of rule pack " + packId + ": " + e.getMessage()); + } + return Collections.emptyList(); + } + private static ClassLoader[] classLoadersToTry(Class anchor) { ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); ClassLoader anchorLoader = anchor != null ? anchor.getClassLoader() : null; diff --git a/plugins/misc/lint/src/test/java/org/apache/hop/lint/LintReportWriterTest.java b/plugins/misc/lint/src/test/java/org/apache/hop/lint/LintReportWriterTest.java index d9b9beae877..f0f27c659e7 100644 --- a/plugins/misc/lint/src/test/java/org/apache/hop/lint/LintReportWriterTest.java +++ b/plugins/misc/lint/src/test/java/org/apache/hop/lint/LintReportWriterTest.java @@ -240,6 +240,50 @@ public void sarifDescribesTheRuleFromAFindingThatCarriesItsDetails() throws Exce assertEquals(0, results.get(1).get("ruleIndex").asInt()); } + /** + * Hop's own verify remarks carry the transform they were reported on as their rule name. The rule + * was described by the first one, so code-scanning tools labelled every finding with it. + * + * @see #9500 + */ + @Test + public void sarifNativeRuleIsNotDescribedByATransformName() throws Exception { + JsonNode rules = + sarifRules( + List.of( + nativeRemark("CONNECTION_NOT_ASSIGNED", "Load customers"), + nativeRemark("CONNECTION_NOT_ASSIGNED", "Load orders"))); + + assertEquals(1, rules.size()); + String text = rules.get(0).get("shortDescription").get("text").asText(); + assertEquals("CONNECTION_NOT_ASSIGNED", text); + } + + /** A remark classified by a native rule is described by that rule's name. */ + @Test + public void sarifNativeRuleTakesTheClassifyingRulesName() throws Exception { + LintRuleDetails details = + new LintRuleDetails("Hop Verify Remark", "Every remark Hop's checks produce", null, null); + JsonNode rules = + sarifRules( + List.of( + nativeRemark("HOP-CHECK", "Load customers").withRuleDetails(details), + nativeRemark("HOP-CHECK", "Load orders").withRuleDetails(details))); + + assertEquals("Hop Verify Remark", rules.get(0).get("shortDescription").get("text").asText()); + } + + private static LintResult nativeRemark(String ruleId, String transformName) { + return new LintResult( + ruleId, + transformName, + "WARNING", + "No connection is set", + PROJECT.resolve("pipelines/load.hpl").toString(), + LintSourceRef.transform(transformName), + LintResult.Origin.HOP_NATIVE); + } + /** The SARIF schema requires a rule's tags to be unique. */ @Test public void sarifTagsAreUnique() throws Exception { diff --git a/plugins/misc/lint/src/test/java/org/apache/hop/lint/registry/RuleRegistryTest.java b/plugins/misc/lint/src/test/java/org/apache/hop/lint/registry/RuleRegistryTest.java index ea23324f06f..2c0d49f9eb0 100644 --- a/plugins/misc/lint/src/test/java/org/apache/hop/lint/registry/RuleRegistryTest.java +++ b/plugins/misc/lint/src/test/java/org/apache/hop/lint/registry/RuleRegistryTest.java @@ -497,6 +497,40 @@ public void overrideDeclarationIgnoresCase() { assertEquals("acme", merged.get("DB-001").getPackId()); } + /** + * A pack registered through ServiceLoader is an {@link AbstractYamlRulePack}; the overrides: in + * its YAML were ignored, so it could not replace a core rule the way a discovered file pack can. + * + * @see #9500 + */ + @Test + public void aClasspathPackMayOverrideACoreRule() { + IHopLintRulePack acme = + new AbstractYamlRulePack( + "acme", + "Acme Rules", + RulePackOwner.VENDOR, + 500, + "org/apache/hop/lint/registry/test-override-pack.yml") {}; + assertEquals(List.of("DB-001"), acme.getOverrides()); + + Map merged = new LinkedHashMap<>(); + RuleRegistry.mergePack(merged, EagerRulePack.of(new HopCoreRulePack())); + assertEquals("hop-core", merged.get("DB-001").getPackId()); + + RuleRegistry.mergePack(merged, EagerRulePack.of(acme)); + + CustomLintRule surviving = merged.get("DB-001"); + assertEquals("acme", surviving.getPackId()); + assertEquals("INFO", surviving.getSeverity()); + } + + /** A classpath pack without an overrides: block declares none. */ + @Test + public void aClasspathPackWithoutOverridesDeclaresNone() { + assertTrue(new HopCoreRulePack().getOverrides().isEmpty()); + } + private CustomLintRule ruleFrom(String packId, String ruleId, String severity) { CustomLintRule rule = new CustomLintRule(); rule.setId(ruleId); diff --git a/plugins/misc/lint/src/test/resources/org/apache/hop/lint/registry/test-override-pack.yml b/plugins/misc/lint/src/test/resources/org/apache/hop/lint/registry/test-override-pack.yml new file mode 100644 index 00000000000..c7db5b3f59a --- /dev/null +++ b/plugins/misc/lint/src/test/resources/org/apache/hop/lint/registry/test-override-pack.yml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# A ServiceLoader-style pack that replaces a core rule, declared in its overrides: block. +pack: + id: acme + name: Acme Rules + owner: VENDOR + priority: 500 + overrides: + - DB-001 + +rules: + DB-001: + type: custom + enabled: true + severity: INFO + target: DATABASE_CONNECTION + targetField: password + condition: NO_HARDCODED + name: "Hardcoded Database Password (Acme)" + description: "Acme's own version of the rule"