Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -168,9 +168,8 @@ private static String renderSarif(
// one rule id while carrying the transform name as their result name, so using that
// here would label the rule after whichever transform happened to be reported first.
rule.put("name", id);
rule.putObject("shortDescription")
.put("text", result.getRuleName() != null ? result.getRuleName() : id);
LintRuleDetails details = result.getRuleDetails();
rule.putObject("shortDescription").put("text", shortDescription(id, result, details));
if (!details.description().isBlank()) {
rule.putObject("fullDescription").put("text", details.description());
}
Expand Down Expand Up @@ -235,6 +234,18 @@ private static void putTags(ObjectNode target, LintRuleDetails details) {
});
}

/**
* A remark from Hop's own checks carries the transform or action it was reported on as its rule
* name, so the first one reported would describe the rule for every finding of it. Those take the
* name of the native rule that classified them, or the rule id.
*/
private static String shortDescription(String id, LintResult result, LintRuleDetails details) {
if (result.getOrigin() == LintResult.Origin.HOP_NATIVE) {
return details.name().isBlank() ? id : details.name();
}
return result.getRuleName() != null ? result.getRuleName() : id;
}

private static String sarifLevel(String severity) {
if ("ERROR".equalsIgnoreCase(severity)) {
return "error";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,38 +27,46 @@
* What a finding carries about the rule that produced it, for the reports to write out.
*
* <p>None of it changes what the rule checks: it is the rule's description, where to read more
* about it, and the tags its pack gave it.
* about it, and the tags its pack gave it, with its name for the reports that describe the rule.
*
* @param name the rule's name, empty when it has none
* @param description the rule's description, empty when it has none
* @param helpUri where to read more about the rule, or null
* @param tags the rule's tags, each key holding one or more values
*/
public record LintRuleDetails(String description, String helpUri, Map<String, List<String>> tags) {
public record LintRuleDetails(
String name, String description, String helpUri, Map<String, List<String>> tags) {

public static final LintRuleDetails NONE = new LintRuleDetails("", null, Map.of());
public static final LintRuleDetails NONE = new LintRuleDetails("", "", null, Map.of());

public LintRuleDetails {
name = name != null ? name : "";
description = description != null ? description : "";
tags = copyOf(tags);
}

public LintRuleDetails(String description, String helpUri, Map<String, List<String>> tags) {
this("", description, helpUri, tags);
}

/** The details of a rule, or {@link #NONE} when there is no rule. */
public static LintRuleDetails of(CustomLintRule rule) {
if (rule == null) {
return NONE;
}
return new LintRuleDetails(rule.getDescription(), rule.getHelpUri(), rule.getTags());
return new LintRuleDetails(
rule.getName(), rule.getDescription(), rule.getHelpUri(), rule.getTags());
}

/**
* Only the tags, for a finding reported under an id other than the rule's own.
*
* <p>A remark from Hop's own checks with an error code of its own is reported under that code.
* The tags of the native rule that classified it apply to it, but that rule's description and
* help link are about the rule, not about the code.
* help link are about the rule, not about the code; so is its name.
*/
public LintRuleDetails tagsOnly() {
return hasTags() ? new LintRuleDetails("", null, tags) : NONE;
return hasTags() ? new LintRuleDetails("", "", null, tags) : NONE;
}

public boolean hasTags() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public abstract class AbstractYamlRulePack implements IHopLintRulePack {
private final RulePackOwner owner;
private final int priority;
private final String resourcePath;
private List<String> overrides;

protected AbstractYamlRulePack(
String packId, String displayName, RulePackOwner owner, int priority, String resourcePath) {
Expand Down Expand Up @@ -57,6 +58,18 @@ public int getPriority() {
return priority;
}

/** The rules this pack replaces, as declared in the {@code overrides:} of its YAML pack block. */
@Override
public synchronized List<String> getOverrides() {
if (overrides == null) {
overrides =
List.copyOf(
YamlRulePackParser.readOverridesWithAdjacentFallback(
resourcePath, packId, owner, getClass()));
}
return overrides;
}

@Override
public List<CustomLintRule> loadRules() {
return YamlRulePackParser.loadRulesWithAdjacentFallback(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,39 @@ public static List<CustomLintRule> loadRulesWithAdjacentFallback(
}
}

/**
* The {@code pack.overrides:} list of a pack shipped as a classpath resource, falling back to the
* YAML beside the plugin jar the same way {@link #loadRulesWithAdjacentFallback} does. A pack
* that cannot be read declares no overrides; loading its rules reports the error.
*/
public static List<String> readOverridesWithAdjacentFallback(
String resourcePath, String packId, RulePackOwner owner, Class<?> anchor) {
if (Utils.isEmpty(resourcePath)) {
return Collections.emptyList();
}
try {
for (ClassLoader loader : classLoadersToTry(anchor)) {
if (loader == null) {
continue;
}
try (InputStream inputStream = loader.getResourceAsStream(resourcePath)) {
if (inputStream != null) {
return parsePackMetadata(readYaml(inputStream, resourcePath), packId, owner, 0)
.overrides();
}
}
}
File adjacent = findPackYamlAdjacentTo(anchor, resourcePath);
if (adjacent != null) {
return readPackMetadata(adjacent, packId, owner, 0).overrides();
}
} catch (IOException | RuntimeException e) {
LogChannel.GENERAL.logError(
"Failed to read the overrides of rule pack " + packId + ": " + e.getMessage());
}
return Collections.emptyList();
}

private static ClassLoader[] classLoadersToTry(Class<?> anchor) {
ClassLoader contextLoader = Thread.currentThread().getContextClassLoader();
ClassLoader anchorLoader = anchor != null ? anchor.getClassLoader() : null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,50 @@ public void sarifDescribesTheRuleFromAFindingThatCarriesItsDetails() throws Exce
assertEquals(0, results.get(1).get("ruleIndex").asInt());
}

/**
* Hop's own verify remarks carry the transform they were reported on as their rule name. The rule
* was described by the first one, so code-scanning tools labelled every finding with it.
*
* @see <a href="https://github.com/apache/hop/issues/9500">#9500</a>
*/
@Test
public void sarifNativeRuleIsNotDescribedByATransformName() throws Exception {
JsonNode rules =
sarifRules(
List.of(
nativeRemark("CONNECTION_NOT_ASSIGNED", "Load customers"),
nativeRemark("CONNECTION_NOT_ASSIGNED", "Load orders")));

assertEquals(1, rules.size());
String text = rules.get(0).get("shortDescription").get("text").asText();
assertEquals("CONNECTION_NOT_ASSIGNED", text);
}

/** A remark classified by a native rule is described by that rule's name. */
@Test
public void sarifNativeRuleTakesTheClassifyingRulesName() throws Exception {
LintRuleDetails details =
new LintRuleDetails("Hop Verify Remark", "Every remark Hop's checks produce", null, null);
JsonNode rules =
sarifRules(
List.of(
nativeRemark("HOP-CHECK", "Load customers").withRuleDetails(details),
nativeRemark("HOP-CHECK", "Load orders").withRuleDetails(details)));

assertEquals("Hop Verify Remark", rules.get(0).get("shortDescription").get("text").asText());
}

private static LintResult nativeRemark(String ruleId, String transformName) {
return new LintResult(
ruleId,
transformName,
"WARNING",
"No connection is set",
PROJECT.resolve("pipelines/load.hpl").toString(),
LintSourceRef.transform(transformName),
LintResult.Origin.HOP_NATIVE);
}

/** The SARIF schema requires a rule's tags to be unique. */
@Test
public void sarifTagsAreUnique() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,40 @@ public void overrideDeclarationIgnoresCase() {
assertEquals("acme", merged.get("DB-001").getPackId());
}

/**
* A pack registered through ServiceLoader is an {@link AbstractYamlRulePack}; the overrides: in
* its YAML were ignored, so it could not replace a core rule the way a discovered file pack can.
*
* @see <a href="https://github.com/apache/hop/issues/9500">#9500</a>
*/
@Test
public void aClasspathPackMayOverrideACoreRule() {
IHopLintRulePack acme =
new AbstractYamlRulePack(
"acme",
"Acme Rules",
RulePackOwner.VENDOR,
500,
"org/apache/hop/lint/registry/test-override-pack.yml") {};
assertEquals(List.of("DB-001"), acme.getOverrides());

Map<String, CustomLintRule> merged = new LinkedHashMap<>();
RuleRegistry.mergePack(merged, EagerRulePack.of(new HopCoreRulePack()));
assertEquals("hop-core", merged.get("DB-001").getPackId());

RuleRegistry.mergePack(merged, EagerRulePack.of(acme));

CustomLintRule surviving = merged.get("DB-001");
assertEquals("acme", surviving.getPackId());
assertEquals("INFO", surviving.getSeverity());
}

/** A classpath pack without an overrides: block declares none. */
@Test
public void aClasspathPackWithoutOverridesDeclaresNone() {
assertTrue(new HopCoreRulePack().getOverrides().isEmpty());
}

private CustomLintRule ruleFrom(String packId, String ruleId, String severity) {
CustomLintRule rule = new CustomLintRule();
rule.setId(ruleId);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# A ServiceLoader-style pack that replaces a core rule, declared in its overrides: block.
pack:
id: acme
name: Acme Rules
owner: VENDOR
priority: 500
overrides:
- DB-001

rules:
DB-001:
type: custom
enabled: true
severity: INFO
target: DATABASE_CONNECTION
targetField: password
condition: NO_HARDCODED
name: "Hardcoded Database Password (Acme)"
description: "Acme's own version of the rule"
Loading