Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,13 @@ public boolean init() {
try {
IValueMeta valueMeta = ValueMetaFactory.createValueMeta(fieldName, field.getType());
valueMeta.setTrimType(field.getTrimType());
// The field's format, as getFields() declares it: text in the YAML is converted with it
valueMeta.setLength(field.getLength());
valueMeta.setPrecision(field.getPrecision());
valueMeta.setConversionMask(field.getFormat());
valueMeta.setDecimalSymbol(field.getDecimalSymbol());
valueMeta.setGroupingSymbol(field.getGroupSymbol());
valueMeta.setCurrencySymbol(field.getCurrencySymbol());
data.rowMeta.addValueMeta(valueMeta);
} catch (Exception e) {
logError("Unable to create value meta", e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,14 @@
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.io.CountingInputStream;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.row.value.ValueMetaNone;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.vfs.HopVfs;
import org.yaml.snakeyaml.Yaml;
Expand Down Expand Up @@ -437,7 +439,9 @@ private Object getValue(Object value, IValueMeta valueMeta) {
case IValueMeta.TYPE_INTEGER -> convertToInteger(normalizedValue);
case IValueMeta.TYPE_NUMBER -> convertToDouble(normalizedValue);
case IValueMeta.TYPE_BIGNUMBER -> convertToBigDecimal(normalizedValue);
case IValueMeta.TYPE_BOOLEAN, IValueMeta.TYPE_DATE, IValueMeta.TYPE_BINARY -> normalizedValue;
case IValueMeta.TYPE_DATE -> convertToDate(normalizedValue, valueMeta);
case IValueMeta.TYPE_BOOLEAN -> convertToBoolean(normalizedValue, valueMeta);
case IValueMeta.TYPE_BINARY -> normalizedValue;
default -> convertToString(normalizedValue, valueMeta);
};
}
Expand Down Expand Up @@ -467,6 +471,39 @@ private static String stripTrailingLineBreaks(String s) {
return s.substring(0, end);
}

/**
* object -> date. SnakeYAML reads an unquoted ISO date as a {@link java.util.Date}; a quoted one,
* or another format, stays text and is parsed with the field's format, so the value matches the
* Date type the field declares.
*/
private Object convertToDate(Object value, IValueMeta valueMeta) {
if (value instanceof java.util.Date) {
return value;
}
return convertText(value, valueMeta);
}

/** object -> boolean: YAML booleans stay, text (Y/N, true/false) is converted. */
private Object convertToBoolean(Object value, IValueMeta valueMeta) {
if (value instanceof Boolean) {
return value;
}
return convertText(value, valueMeta);
}

/** Text to the field's type with its format, as Hop converts a String field. */
private static Object convertText(Object value, IValueMeta valueMeta) {
ValueMetaString text = new ValueMetaString(valueMeta.getName());
text.setConversionMask(valueMeta.getConversionMask());
text.setDateFormatLenient(valueMeta.isDateFormatLenient());
try {
return valueMeta.convertData(text, value.toString());
} catch (HopValueException e) {
throw new IllegalArgumentException(
"Unable to convert '" + value + "' of field '" + valueMeta.getName() + "'", e);
}
}

/** object -> long */
@SuppressWarnings("javabugs:S2259") // the value is never null when this is reached
private Long convertToInteger(Object value) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@
import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.row.value.ValueMetaBoolean;
import org.apache.hop.core.row.value.ValueMetaDate;
import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.junit.rules.RestoreHopEnvironmentExtension;
import org.junit.jupiter.api.BeforeAll;
Expand Down Expand Up @@ -102,6 +105,29 @@ void testLoadString_ListOfMap() {
assertInstanceOf(Date.class, row[2]);
}

@Test
void aQuotedDateBecomesTheDateItsFieldDeclares() {
String yaml = """
order_id: 1
order_date: '2020-02-28'
shipped: 'Y'
""";

reader.loadString(yaml);

RowMeta rowMeta = new RowMeta();
rowMeta.addValueMeta(new ValueMetaInteger("order_id"));
ValueMetaDate date = new ValueMetaDate("order_date");
date.setConversionMask("yyyy-MM-dd");
rowMeta.addValueMeta(date);
rowMeta.addValueMeta(new ValueMetaBoolean("shipped"));
Object[] row = reader.getRow(rowMeta);

assertEquals(1L, row[0]);
assertInstanceOf(Date.class, row[1]);
assertEquals(true, row[2]);
}

@Test
void testLoadString_ListMultipleValues() {
String yaml = """
Expand Down
Loading