Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@
@SuppressWarnings("java:S1104")
public class Calculator extends BaseTransform<CalculatorMeta, CalculatorData> {

/** Per field: its metadata in normal storage, for a lazily converted field. */
private IValueMeta[] normalStorageMetas;

private static final Class<?> PKG = CalculatorMeta.class;
public static final String CONST_FOR_CALCULATION = " for calculation #";

Expand Down Expand Up @@ -196,6 +199,35 @@ public boolean processRow() throws HopException {
return true;
}

/**
* The metadata of a field as a calculation sees it: in normal storage. A lazily converted field
* (CSV input with lazy conversion) keeps its raw bytes in the row that goes on, but a calculation
* works on the value, so it gets the value and metadata in normal storage.
*/
private IValueMeta normalStorageMeta(int index) {
IValueMeta valueMeta = data.getCalcRowMeta().getValueMeta(index);
if (valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_NORMAL) {
return valueMeta;
}
if (normalStorageMetas == null) {
normalStorageMetas = new IValueMeta[data.getCalcRowMeta().size()];
}
if (normalStorageMetas[index] == null) {
IValueMeta normal = valueMeta.clone();
normal.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
normalStorageMetas[index] = normal;
}
return normalStorageMetas[index];
}

private static Object toNormalStorage(IValueMeta valueMeta, Object value)
throws HopValueException {
if (value == null || valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_NORMAL) {
return value;
}
return valueMeta.convertToNormalStorageType(value);
}

/**
* @param inputRowMeta the input row metadata
* @param r the input row (data)
Expand All @@ -216,24 +248,27 @@ private Object[] calcFields(IRowMeta inputRowMeta, Object[] r)
Object dataA = null;

if (data.getFieldIndexes()[i].indexA >= 0) {
metaA = data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexA);
dataA = calcData[data.getFieldIndexes()[i].indexA];
int indexA = data.getFieldIndexes()[i].indexA;
metaA = normalStorageMeta(indexA);
dataA = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexA), calcData[indexA]);
}

IValueMeta metaB = null;
Object dataB = null;

if (data.getFieldIndexes()[i].indexB >= 0) {
metaB = data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexB);
dataB = calcData[data.getFieldIndexes()[i].indexB];
int indexB = data.getFieldIndexes()[i].indexB;
metaB = normalStorageMeta(indexB);
dataB = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexB), calcData[indexB]);
}

IValueMeta metaC = null;
Object dataC = null;

if (data.getFieldIndexes()[i].indexC >= 0) {
metaC = data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexC);
dataC = calcData[data.getFieldIndexes()[i].indexC];
int indexC = data.getFieldIndexes()[i].indexC;
metaC = normalStorageMeta(indexC);
dataC = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexC), calcData[indexC]);
}

CalculationType calcType = fn.getCalcType();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@

import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.hop.core.Const;
import org.apache.hop.core.HopEnvironment;
Expand Down Expand Up @@ -1015,4 +1017,103 @@ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row)
fail();
}
}

/**
* After a CSV file input with lazy conversion the fields hold the bytes read from the file. The
* calculations get the values, as they do without lazy conversion.
*/
@Test
void lazilyConvertedOperandsGiveTheResultsOfConvertedOnes() throws HopException {
String[][] text = {{"MiXed Case", "3", "4"}, {"", "", "5"}};

List<List<Object>> converted = calculate(text, false);
List<List<Object>> lazy = calculate(text, true);

assertEquals("mixed case", converted.get(0).get(0));
assertEquals(7L, converted.get(0).get(1));
assertEquals(converted, lazy);
}

/** Lower case of the text field and the sum of the two numbers, for every row. */
private List<List<Object>> calculate(String[][] text, boolean lazy) throws HopException {
RowMeta inputRowMeta = new RowMeta();
inputRowMeta.addValueMeta(new ValueMetaString("text"));
inputRowMeta.addValueMeta(new ValueMetaInteger("a"));
inputRowMeta.addValueMeta(new ValueMetaInteger("b"));
for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) {
// As CSV file input reads a field with lazy conversion
valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING);
valueMeta.setStorageMetadata(ValueMetaFactory.createParseMeta(valueMeta));
}
Object[][] rows = new Object[text.length][];
for (int r = 0; r < text.length; r++) {
rows[r] = new Object[text[r].length];
for (int f = 0; f < text[r].length; f++) {
byte[] bytes = text[r][f].getBytes(StandardCharsets.UTF_8);
rows[r][f] = lazy ? bytes : inputRowMeta.getValueMeta(f).convertToNormalStorageType(bytes);
}
}
if (!lazy) {
for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) {
valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
}
}
IRowSet inputRowSet = smh.getMockInputRowSet(rows);
inputRowSet.setRowMeta(inputRowMeta);

CalculatorMeta meta = new CalculatorMeta();
meta.getFunctions()
.add(
new CalculatorMetaFunction(
"lower",
CalculationType.LOWER_CASE,
"text",
null,
null,
"String",
0,
0,
"",
"",
"",
"",
false));
meta.getFunctions()
.add(
new CalculatorMetaFunction(
"sum",
CalculationType.ADD,
"a",
"b",
null,
"Integer",
0,
0,
"",
"",
"",
"",
false));

Calculator calculator =
new Calculator(
smh.transformMeta, meta, new CalculatorData(), 0, smh.pipelineMeta, smh.pipeline);
calculator.addRowSetToInputRowSets(inputRowSet);
calculator.setInputRowMeta(inputRowMeta);
calculator.init();

List<List<Object>> results = new ArrayList<>();
calculator.addRowListener(
new RowAdapter() {
@Override
public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
results.add(Arrays.asList(row[3], row[4]));
}
});
while (calculator.processRow()) {
// one row per call
}
assertEquals(text.length, results.size());
return results;
}
}
Loading