Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1395,6 +1395,13 @@ private void getTransformFieldsRegular(
row = getThisTransformFields(variables, transformMeta, targetTransform, row, monitor);
}

// The transform's own values of an error row, a count for example
if (transformMeta.getTransform() != null) {
transformMeta
.getTransform()
.getErrorRowFields(row, transformMeta.getName(), variables, metadataProvider);
}

// Add to this the error fields...
TransformErrorMeta transformErrorMeta = transformMeta.getTransformErrorMeta();
row.addRowMeta(transformErrorMeta.getErrorRowMeta(variables));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,23 @@ SqlStatement getSqlStatements(
*/
boolean supportsErrorHandling();

/**
* The fields of the rows this transform sends to its error hop, before the error fields. They are
* the input fields, which this method receives; a transform that sends other values with an error
* row, a count for example, adds or changes them here.
*
* @param rowMeta the input fields, to change into the fields of an error row
* @param name the name of the transform, the origin of a field it adds
* @param variables the variables to resolve settings with
* @param metadataProvider the metadata the transform can refer to
* @throws HopTransformException when the fields can't be worked out
*/
default void getErrorRowFields(
IRowMeta rowMeta, String name, IVariables variables, IHopMetadataProvider metadataProvider)
throws HopTransformException {
// An error row holds the input row
}

/**
* @return true if this transform supports multi-copies execution. By default return true.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,13 +191,29 @@ public void getFields(
}
}
if (countRows) {
IValueMeta v = new ValueMetaInteger(countField);
v.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0);
v.setOrigin(name);
row.addValueMeta(v);
row.addValueMeta(countValueMeta(name));
}
}

/**
* A duplicate goes to the error hop with the count of its set so far, in the count field, after
* the input fields.
*/
@Override
public void getErrorRowFields(
IRowMeta row, String name, IVariables variables, IHopMetadataProvider metadataProvider) {
if (countRows) {
row.addValueMeta(countValueMeta(name));
}
}

private IValueMeta countValueMeta(String name) {
IValueMeta v = new ValueMetaInteger(countField);
v.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0);
v.setOrigin(name);
return v;
}

@Override
public void check(
List<ICheckResult> remarks,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hop.pipeline.transforms.uniquerows;

import static org.junit.jupiter.api.Assertions.assertEquals;

import java.util.List;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineHopMeta;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.ITransform;
import org.apache.hop.pipeline.transform.ITransformData;
import org.apache.hop.pipeline.transform.TransformErrorMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

/** The fields a pipeline declares for the error hop of Unique rows are the ones it writes. */
class UniqueRowsErrorHopFieldsTest {

@RegisterExtension
static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension();

/** A duplicate goes to the error hop with the count of its set after the input fields. */
@Test
void theErrorHopDeclaresTheCountAfterTheInputFields() throws Exception {
UniqueRowsMeta uniqueRowsMeta = new UniqueRowsMeta();
uniqueRowsMeta.setCountRows(true);
uniqueRowsMeta.setCountField("n");
// Duplicates go to the error hop
uniqueRowsMeta.setRejectDuplicateRow(true);

TransformMeta input = new TransformMeta("input", new CodeInputMeta());
TransformMeta unique = new TransformMeta("unique", uniqueRowsMeta);
TransformMeta duplicates = new TransformMeta("duplicates", new CodeInputMeta());
TransformErrorMeta errorMeta = new TransformErrorMeta(unique, duplicates);
errorMeta.setEnabled(true);
unique.setTransformErrorMeta(errorMeta);

PipelineMeta pipelineMeta = new PipelineMeta();
pipelineMeta.addTransform(input);
pipelineMeta.addTransform(unique);
pipelineMeta.addTransform(duplicates);
pipelineMeta.addPipelineHop(new PipelineHopMeta(input, unique));
pipelineMeta.addPipelineHop(new PipelineHopMeta(unique, duplicates));

IRowMeta errorRowFields =
pipelineMeta.getTransformFields(new Variables(), unique, duplicates, null);

assertEquals(List.of("code", "n"), List.of(errorRowFields.getFieldNames()));
}

/** A transform that gives one field, as an input would. */
public static class CodeInputMeta extends BaseTransformMeta<ITransform, ITransformData> {
@Override
public void getFields(
IRowMeta row,
String name,
IRowMeta[] info,
TransformMeta nextTransform,
IVariables variables,
IHopMetadataProvider metadataProvider) {
row.addValueMeta(new ValueMetaString("code"));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,20 @@

package org.apache.hop.pipeline.transforms.uniquerows;

import static org.junit.jupiter.api.Assertions.assertEquals;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
import org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator;
Expand Down Expand Up @@ -84,4 +92,24 @@ public boolean validateTestObject(UniqueField testObject, Object actual) {
&& testObject.isCaseInsensitive() == actualObject.isCaseInsensitive();
}
}

/** A duplicate goes to the error hop with its count when the transform counts. */
@Test
void testErrorRowFieldsHoldTheCount() throws Exception {
UniqueRowsMeta meta = new UniqueRowsMeta();
meta.setCountField("n");
IRowMeta row = new RowMeta();
row.addValueMeta(new ValueMetaString("code"));

meta.setCountRows(false);
IRowMeta withoutCount = row.clone();
meta.getErrorRowFields(withoutCount, "unique", new Variables(), null);
assertEquals(List.of("code"), List.of(withoutCount.getFieldNames()));

meta.setCountRows(true);
IRowMeta withCount = row.clone();
meta.getErrorRowFields(withCount, "unique", new Variables(), null);
assertEquals(List.of("code", "n"), List.of(withCount.getFieldNames()));
assertEquals(IValueMeta.TYPE_INTEGER, withCount.getValueMeta(1).getType());
}
}
Loading