diff --git a/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java b/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java index ff0c438457f..8b21d17b0ed 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java +++ b/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java @@ -1395,6 +1395,13 @@ private void getTransformFieldsRegular( row = getThisTransformFields(variables, transformMeta, targetTransform, row, monitor); } + // The transform's own values of an error row, a count for example + if (transformMeta.getTransform() != null) { + transformMeta + .getTransform() + .getErrorRowFields(row, transformMeta.getName(), variables, metadataProvider); + } + // Add to this the error fields... TransformErrorMeta transformErrorMeta = transformMeta.getTransformErrorMeta(); row.addRowMeta(transformErrorMeta.getErrorRowMeta(variables)); diff --git a/engine/src/main/java/org/apache/hop/pipeline/transform/ITransformMeta.java b/engine/src/main/java/org/apache/hop/pipeline/transform/ITransformMeta.java index b75c0c52eb1..23e9c9b4c29 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/transform/ITransformMeta.java +++ b/engine/src/main/java/org/apache/hop/pipeline/transform/ITransformMeta.java @@ -343,6 +343,23 @@ SqlStatement getSqlStatements( */ boolean supportsErrorHandling(); + /** + * The fields of the rows this transform sends to its error hop, before the error fields. They are + * the input fields, which this method receives; a transform that sends other values with an error + * row, a count for example, adds or changes them here. + * + * @param rowMeta the input fields, to change into the fields of an error row + * @param name the name of the transform, the origin of a field it adds + * @param variables the variables to resolve settings with + * @param metadataProvider the metadata the transform can refer to + * @throws HopTransformException when the fields can't be worked out + */ + default void getErrorRowFields( + IRowMeta rowMeta, String name, IVariables variables, IHopMetadataProvider metadataProvider) + throws HopTransformException { + // An error row holds the input row + } + /** * @return true if this transform supports multi-copies execution. By default return true. */ diff --git a/plugins/transforms/uniquerows/src/main/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsMeta.java b/plugins/transforms/uniquerows/src/main/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsMeta.java index 2cc78415748..fd68fe4eea5 100644 --- a/plugins/transforms/uniquerows/src/main/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsMeta.java +++ b/plugins/transforms/uniquerows/src/main/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsMeta.java @@ -191,13 +191,29 @@ public void getFields( } } if (countRows) { - IValueMeta v = new ValueMetaInteger(countField); - v.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0); - v.setOrigin(name); - row.addValueMeta(v); + row.addValueMeta(countValueMeta(name)); } } + /** + * A duplicate goes to the error hop with the count of its set so far, in the count field, after + * the input fields. + */ + @Override + public void getErrorRowFields( + IRowMeta row, String name, IVariables variables, IHopMetadataProvider metadataProvider) { + if (countRows) { + row.addValueMeta(countValueMeta(name)); + } + } + + private IValueMeta countValueMeta(String name) { + IValueMeta v = new ValueMetaInteger(countField); + v.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0); + v.setOrigin(name); + return v; + } + @Override public void check( List remarks, diff --git a/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsErrorHopFieldsTest.java b/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsErrorHopFieldsTest.java new file mode 100644 index 00000000000..c34980d258f --- /dev/null +++ b/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsErrorHopFieldsTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.uniquerows; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineHopMeta; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.ITransform; +import org.apache.hop.pipeline.transform.ITransformData; +import org.apache.hop.pipeline.transform.TransformErrorMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** The fields a pipeline declares for the error hop of Unique rows are the ones it writes. */ +class UniqueRowsErrorHopFieldsTest { + + @RegisterExtension + static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); + + /** A duplicate goes to the error hop with the count of its set after the input fields. */ + @Test + void theErrorHopDeclaresTheCountAfterTheInputFields() throws Exception { + UniqueRowsMeta uniqueRowsMeta = new UniqueRowsMeta(); + uniqueRowsMeta.setCountRows(true); + uniqueRowsMeta.setCountField("n"); + // Duplicates go to the error hop + uniqueRowsMeta.setRejectDuplicateRow(true); + + TransformMeta input = new TransformMeta("input", new CodeInputMeta()); + TransformMeta unique = new TransformMeta("unique", uniqueRowsMeta); + TransformMeta duplicates = new TransformMeta("duplicates", new CodeInputMeta()); + TransformErrorMeta errorMeta = new TransformErrorMeta(unique, duplicates); + errorMeta.setEnabled(true); + unique.setTransformErrorMeta(errorMeta); + + PipelineMeta pipelineMeta = new PipelineMeta(); + pipelineMeta.addTransform(input); + pipelineMeta.addTransform(unique); + pipelineMeta.addTransform(duplicates); + pipelineMeta.addPipelineHop(new PipelineHopMeta(input, unique)); + pipelineMeta.addPipelineHop(new PipelineHopMeta(unique, duplicates)); + + IRowMeta errorRowFields = + pipelineMeta.getTransformFields(new Variables(), unique, duplicates, null); + + assertEquals(List.of("code", "n"), List.of(errorRowFields.getFieldNames())); + } + + /** A transform that gives one field, as an input would. */ + public static class CodeInputMeta extends BaseTransformMeta { + @Override + public void getFields( + IRowMeta row, + String name, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) { + row.addValueMeta(new ValueMetaString("code")); + } + } +} diff --git a/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsMetaTest.java b/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsMetaTest.java index 00fa05eace1..b53830dfb06 100644 --- a/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsMetaTest.java +++ b/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsMetaTest.java @@ -17,12 +17,20 @@ package org.apache.hop.pipeline.transforms.uniquerows; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Random; import java.util.UUID; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.Variables; import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester; import org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator; @@ -84,4 +92,24 @@ public boolean validateTestObject(UniqueField testObject, Object actual) { && testObject.isCaseInsensitive() == actualObject.isCaseInsensitive(); } } + + /** A duplicate goes to the error hop with its count when the transform counts. */ + @Test + void testErrorRowFieldsHoldTheCount() throws Exception { + UniqueRowsMeta meta = new UniqueRowsMeta(); + meta.setCountField("n"); + IRowMeta row = new RowMeta(); + row.addValueMeta(new ValueMetaString("code")); + + meta.setCountRows(false); + IRowMeta withoutCount = row.clone(); + meta.getErrorRowFields(withoutCount, "unique", new Variables(), null); + assertEquals(List.of("code"), List.of(withoutCount.getFieldNames())); + + meta.setCountRows(true); + IRowMeta withCount = row.clone(); + meta.getErrorRowFields(withCount, "unique", new Variables(), null); + assertEquals(List.of("code", "n"), List.of(withCount.getFieldNames())); + assertEquals(IValueMeta.TYPE_INTEGER, withCount.getValueMeta(1).getType()); + } }