diff --git a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java index e3658443a16..a0589803162 100644 --- a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java +++ b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java @@ -37,6 +37,9 @@ @SuppressWarnings("java:S1104") public class Calculator extends BaseTransform { + /** Per field: its metadata in normal storage, for a lazily converted field. */ + private IValueMeta[] normalStorageMetas; + private static final Class PKG = CalculatorMeta.class; public static final String CONST_FOR_CALCULATION = " for calculation #"; @@ -196,6 +199,35 @@ public boolean processRow() throws HopException { return true; } + /** + * The metadata of a field as a calculation sees it: in normal storage. A lazily converted field + * (CSV input with lazy conversion) keeps its raw bytes in the row that goes on, but a calculation + * works on the value, so it gets the value and metadata in normal storage. + */ + private IValueMeta normalStorageMeta(int index) { + IValueMeta valueMeta = data.getCalcRowMeta().getValueMeta(index); + if (valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_NORMAL) { + return valueMeta; + } + if (normalStorageMetas == null) { + normalStorageMetas = new IValueMeta[data.getCalcRowMeta().size()]; + } + if (normalStorageMetas[index] == null) { + IValueMeta normal = valueMeta.clone(); + normal.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL); + normalStorageMetas[index] = normal; + } + return normalStorageMetas[index]; + } + + private static Object toNormalStorage(IValueMeta valueMeta, Object value) + throws HopValueException { + if (value == null || valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_NORMAL) { + return value; + } + return valueMeta.convertToNormalStorageType(value); + } + /** * @param inputRowMeta the input row metadata * @param r the input row (data) @@ -216,24 +248,27 @@ private Object[] calcFields(IRowMeta inputRowMeta, Object[] r) Object dataA = null; if (data.getFieldIndexes()[i].indexA >= 0) { - metaA = data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexA); - dataA = calcData[data.getFieldIndexes()[i].indexA]; + int indexA = data.getFieldIndexes()[i].indexA; + metaA = normalStorageMeta(indexA); + dataA = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexA), calcData[indexA]); } IValueMeta metaB = null; Object dataB = null; if (data.getFieldIndexes()[i].indexB >= 0) { - metaB = data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexB); - dataB = calcData[data.getFieldIndexes()[i].indexB]; + int indexB = data.getFieldIndexes()[i].indexB; + metaB = normalStorageMeta(indexB); + dataB = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexB), calcData[indexB]); } IValueMeta metaC = null; Object dataC = null; if (data.getFieldIndexes()[i].indexC >= 0) { - metaC = data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexC); - dataC = calcData[data.getFieldIndexes()[i].indexC]; + int indexC = data.getFieldIndexes()[i].indexC; + metaC = normalStorageMeta(indexC); + dataC = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexC), calcData[indexC]); } CalculationType calcType = fn.getCalcType(); diff --git a/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java b/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java index 727ccf767cb..ba20cc09be4 100644 --- a/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java +++ b/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java @@ -30,9 +30,11 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.hop.core.Const; import org.apache.hop.core.HopEnvironment; @@ -1015,4 +1017,103 @@ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) fail(); } } + + /** + * After a CSV file input with lazy conversion the fields hold the bytes read from the file. The + * calculations get the values, as they do without lazy conversion. + */ + @Test + void lazilyConvertedOperandsGiveTheResultsOfConvertedOnes() throws HopException { + String[][] text = {{"MiXed Case", "3", "4"}, {"", "", "5"}}; + + List> converted = calculate(text, false); + List> lazy = calculate(text, true); + + assertEquals("mixed case", converted.get(0).get(0)); + assertEquals(7L, converted.get(0).get(1)); + assertEquals(converted, lazy); + } + + /** Lower case of the text field and the sum of the two numbers, for every row. */ + private List> calculate(String[][] text, boolean lazy) throws HopException { + RowMeta inputRowMeta = new RowMeta(); + inputRowMeta.addValueMeta(new ValueMetaString("text")); + inputRowMeta.addValueMeta(new ValueMetaInteger("a")); + inputRowMeta.addValueMeta(new ValueMetaInteger("b")); + for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) { + // As CSV file input reads a field with lazy conversion + valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING); + valueMeta.setStorageMetadata(ValueMetaFactory.createParseMeta(valueMeta)); + } + Object[][] rows = new Object[text.length][]; + for (int r = 0; r < text.length; r++) { + rows[r] = new Object[text[r].length]; + for (int f = 0; f < text[r].length; f++) { + byte[] bytes = text[r][f].getBytes(StandardCharsets.UTF_8); + rows[r][f] = lazy ? bytes : inputRowMeta.getValueMeta(f).convertToNormalStorageType(bytes); + } + } + if (!lazy) { + for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) { + valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL); + } + } + IRowSet inputRowSet = smh.getMockInputRowSet(rows); + inputRowSet.setRowMeta(inputRowMeta); + + CalculatorMeta meta = new CalculatorMeta(); + meta.getFunctions() + .add( + new CalculatorMetaFunction( + "lower", + CalculationType.LOWER_CASE, + "text", + null, + null, + "String", + 0, + 0, + "", + "", + "", + "", + false)); + meta.getFunctions() + .add( + new CalculatorMetaFunction( + "sum", + CalculationType.ADD, + "a", + "b", + null, + "Integer", + 0, + 0, + "", + "", + "", + "", + false)); + + Calculator calculator = + new Calculator( + smh.transformMeta, meta, new CalculatorData(), 0, smh.pipelineMeta, smh.pipeline); + calculator.addRowSetToInputRowSets(inputRowSet); + calculator.setInputRowMeta(inputRowMeta); + calculator.init(); + + List> results = new ArrayList<>(); + calculator.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) { + results.add(Arrays.asList(row[3], row[4])); + } + }); + while (calculator.processRow()) { + // one row per call + } + assertEquals(text.length, results.size()); + return results; + } }