diff --git a/engine/src/main/java/org/apache/hop/pipeline/SingleThreadedPipelineExecutor.java b/engine/src/main/java/org/apache/hop/pipeline/SingleThreadedPipelineExecutor.java index 5257f2017ed..12550973871 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/SingleThreadedPipelineExecutor.java +++ b/engine/src/main/java/org/apache/hop/pipeline/SingleThreadedPipelineExecutor.java @@ -299,6 +299,19 @@ public boolean init() { * * @return true if more iterations can be performed. False if this is not the case. */ + /** + * Whether every input row set is done and has no rows left: the transform has read all its input. + * A transform removes a row set it read to the end, so no row sets left means the same. + */ + private static boolean isAllInputRead(List rowSets) { + for (IRowSet rowSet : rowSets) { + if (!rowSet.isDone() || rowSet.size() > 0) { + return false; + } + } + return true; + } + public boolean oneIteration() throws HopException { this.exceptionsRaisedCounter = 0; @@ -369,6 +382,21 @@ public boolean oneIteration() throws HopException { return false; } } + + // All input arrived and was read, but the transform can still have work: a reader + // going through the file a row named, a mapping, the rows a Table output still holds. + // Like a transform without input, it is called until it says it is done. Otherwise it + // never sees the end of its input, and the single-threaded engine, which runs one + // iteration, disposes it with that work undone. + // + if (!transformDone && isAllInputRead(combi.transform.getInputRowSets())) { + while (!transformDone && !pipeline.isStopped()) { + transformDone = !combi.transform.processRow(); + if (combi.transform.getErrors() > 0) { + return false; + } + } + } } // Signal the transform that a batch of rows has passed for this iteration (sort rows and diff --git a/integration-tests/kafka/0006-kafka-consumer-table-output-subpipeline.hpl b/integration-tests/kafka/0006-kafka-consumer-table-output-subpipeline.hpl new file mode 100644 index 00000000000..d5b7559cab4 --- /dev/null +++ b/integration-tests/kafka/0006-kafka-consumer-table-output-subpipeline.hpl @@ -0,0 +1,167 @@ + + + + + 0006-kafka-consumer-table-output-subpipeline + Y + Every batch of 0006-kafka-consumer-table-output goes through Table output; the rows after it go back to the consumer + + + Normal + + + N + 1000 + 100 + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + + Injector + Table output + Y + + + Table output + Out record + Y + + + + Injector + Injector + + Y + + 1 + + none + + + + + key + String + -1 + -1 + + + message + String + -1 + -1 + + + topic + String + -1 + -1 + + + partition + Integer + -1 + -1 + + + offset + Integer + -1 + -1 + + + timestamp + Integer + -1 + -1 + + + + + 96 + 96 + + + + Table output + TableOutput + + Y + + 1 + + none + + + 1000 + h2-kafka + + + message + message + + + + N + N + N + + Y + N + + Y + kafka_table_output
+ + N + Y + N + Y + + + 288 + 96 + +
+ + Out record + Dummy + + Y + + 1 + + none + + + + + 480 + 96 + + + + + +
diff --git a/integration-tests/kafka/0006-kafka-consumer-table-output-validation.hpl b/integration-tests/kafka/0006-kafka-consumer-table-output-validation.hpl new file mode 100644 index 00000000000..bafaff636f6 --- /dev/null +++ b/integration-tests/kafka/0006-kafka-consumer-table-output-validation.hpl @@ -0,0 +1,337 @@ + + + + + 0006-kafka-consumer-table-output-validation + Y + Verify 0006-kafka-consumer-table-output: 20 rows came back after Table output and 20 rows are in the table + + + Normal + + + N + 1000 + 100 + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + + rows after Table output + count rows + Y + + + count rows + rows: n <> 20 + Y + + + rows: n <> 20 + rows: Abort + Y + + + rows in the table + count table rows + Y + + + count table rows + table: n <> 20 + Y + + + table: n <> 20 + table: Abort + Y + + + + rows after Table output + RowsFromResult + + Y + + 1 + + none + + + + + key + String + -1 + -1 + + + message + String + -1 + -1 + + + topic + String + -1 + -1 + + + partition + Integer + -1 + -1 + + + offset + Integer + -1 + -1 + + + timestamp + Integer + -1 + -1 + + + + + 96 + 96 + + + + count rows + GroupBy + + Y + + 1 + + none + + + N + N + + ${java.io.tmpdir} + grp + N + + Y + + + + + n + message + COUNT_ALL + + + + + + 288 + 96 + + + + rows: n <> 20 + FilterRows + + Y + + 1 + + none + + + + + + + N + n + <> + + + constant + Integer + 20 + -1 + 0 + N + ####0;-####0 + + + + + + 480 + 96 + + + + rows: Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0006-kafka-consumer-table-output: not every row came back from the sub-pipeline after Table output + 0 + + + 672 + 96 + + + + rows in the table + TableInput + + Y + + 1 + + none + + + h2-kafka + SELECT message FROM kafka_table_output + 0 + + N + N + + + 96 + 224 + + + + count table rows + GroupBy + + Y + + 1 + + none + + + N + N + + ${java.io.tmpdir} + grp + N + + Y + + + + + n + message + COUNT_ALL + + + + + + 288 + 224 + + + + table: n <> 20 + FilterRows + + Y + + 1 + + none + + + + + + + N + n + <> + + + constant + Integer + 20 + -1 + 0 + N + ####0;-####0 + + + + + + 480 + 224 + + + + table: Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0006-kafka-consumer-table-output: not every row was written to kafka_table_output + 0 + + + 672 + 224 + + + + + + diff --git a/integration-tests/kafka/0006-kafka-consumer-table-output.hpl b/integration-tests/kafka/0006-kafka-consumer-table-output.hpl new file mode 100644 index 00000000000..74f17116ee4 --- /dev/null +++ b/integration-tests/kafka/0006-kafka-consumer-table-output.hpl @@ -0,0 +1,103 @@ + + + + + 0006-kafka-consumer-table-output + Y + A Kafka consumer whose sub-pipeline writes through Table output, 20 messages in batches of 5: every row must come back after Table output (#9526) + + + Normal + + + N + 1000 + 100 + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + + Kafka Consumer + Copy rows to result + Y + + + + Kafka Consumer + KafkaConsumer + + Y + + 1 + + none + + + hop-test-table-output + hop-it-table-output + ${PROJECT_HOME}/0006-kafka-consumer-table-output-subpipeline.hpl + Out record + 5 + 1000 + Y + 5000 + ${BOOTSTRAP_SERVERS} + N + key + message + topic + partition + offset + timestamp + + + + + 96 + 96 + + + + Copy rows to result + RowsToResult + + Y + + 1 + + none + + + + + 288 + 96 + + + + + + diff --git a/integration-tests/kafka/main-0006-kafka-consumer-table-output.hwf b/integration-tests/kafka/main-0006-kafka-consumer-table-output.hwf new file mode 100644 index 00000000000..0714c18b5d3 --- /dev/null +++ b/integration-tests/kafka/main-0006-kafka-consumer-table-output.hwf @@ -0,0 +1,206 @@ + + + + main-0006-kafka-consumer-table-output + Y + + + - + - + 2021/12/30 10:42:13.629 + 2021/12/30 10:42:13.629 + + + + BOOTSTRAP_SERVERS + Kafka bootstrap servers (docker IT network hostname) + kafka:9092 + + + + + N + 0 + 0 + 60 + 1 + 1 + 0 + 12 + N + Start + + SPECIAL + + 128 + 144 + N + + + + empty table + + SQL + + h2-kafka + N + CREATE TABLE IF NOT EXISTS kafka_table_output (message VARCHAR(100)); +DELETE FROM kafka_table_output; + N + N + N + 0 + 224 + 96 + + + + prepare-kafka-test-0006-table-output.hpl + + PIPELINE + + ${PROJECT_HOME}/prepare-kafka-test-0006-table-output.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 0 + 352 + 96 + + + + 0006-kafka-consumer-table-output.hpl + + PIPELINE + + ${PROJECT_HOME}/0006-kafka-consumer-table-output.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 0 + 480 + 96 + + + + 0006-kafka-consumer-table-output-validation.hpl + + PIPELINE + + ${PROJECT_HOME}/0006-kafka-consumer-table-output-validation.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 0 + 608 + 96 + + + + + + Start + empty table + 0 + 0 + Y + Y + Y + + + empty table + prepare-kafka-test-0006-table-output.hpl + 0 + 0 + Y + Y + N + + + prepare-kafka-test-0006-table-output.hpl + 0006-kafka-consumer-table-output.hpl + 0 + 0 + Y + Y + N + + + 0006-kafka-consumer-table-output.hpl + 0006-kafka-consumer-table-output-validation.hpl + 0 + 0 + Y + Y + N + + + + + + diff --git a/integration-tests/kafka/metadata/rdbms/h2-kafka.json b/integration-tests/kafka/metadata/rdbms/h2-kafka.json new file mode 100644 index 00000000000..434ed7e32a0 --- /dev/null +++ b/integration-tests/kafka/metadata/rdbms/h2-kafka.json @@ -0,0 +1,26 @@ +{ + "rdbms": { + "H2": { + "databaseName": "kafka", + "pluginId": "H2", + "accessType": 0, + "hostname": "", + "password": "Encrypted ", + "pluginName": "H2", + "port": "", + "attributes": { + "SUPPORTS_TIMESTAMP_DATA_TYPE": "N", + "QUOTE_ALL_FIELDS": "N", + "SUPPORTS_BOOLEAN_DATA_TYPE": "N", + "FORCE_IDENTIFIERS_TO_LOWERCASE": "N", + "PRESERVE_RESERVED_WORD_CASE": "Y", + "SQL_CONNECT": "", + "FORCE_IDENTIFIERS_TO_UPPERCASE": "N", + "PREFERRED_SCHEMA_NAME": "" + }, + "manualUrl": "jdbc:h2:mem:kafka_it;DB_CLOSE_DELAY=-1", + "username": "" + } + }, + "name": "h2-kafka" +} diff --git a/integration-tests/kafka/prepare-kafka-test-0006-table-output.hpl b/integration-tests/kafka/prepare-kafka-test-0006-table-output.hpl new file mode 100644 index 00000000000..510532f65ed --- /dev/null +++ b/integration-tests/kafka/prepare-kafka-test-0006-table-output.hpl @@ -0,0 +1,107 @@ + + + + + prepare-kafka-test-0006-table-output + Y + Sends 20 messages for 0006-kafka-consumer-table-output + + + Normal + + + N + 1000 + 100 + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + + Generate rows + Kafka Producer + Y + + + + Generate rows + RowGenerator + + Y + + 1 + + none + + + + + -1 + field + -1 + N + String + Hello Hop! + + + 5000 + FiveSecondsAgo + N + 20 + now + + + 96 + 96 + + + + Kafka Producer + KafkaProducerOutput + + Y + + 1 + + none + + + ${BOOTSTRAP_SERVERS} + hop-test-table-output + + + field + + + + + 288 + 96 + + + + + + diff --git a/integration-tests/single_threaded/0005-csv-filename-from-field-validation.hpl b/integration-tests/single_threaded/0005-csv-filename-from-field-validation.hpl new file mode 100644 index 00000000000..b7d9cdb9cc9 --- /dev/null +++ b/integration-tests/single_threaded/0005-csv-filename-from-field-validation.hpl @@ -0,0 +1,282 @@ + + + + + 0005-csv-filename-from-field-validation + Y + Verify the result rows of 0005-csv-filename-from-field, which ran on the single threaded engine + + + Normal + + + N + 1000 + 100 + - + 2026/11/06 12:00:00.000 + - + 2026/11/06 12:00:00.000 + + + + + + Expected + Merge rows (diff) + Y + + + Output of the test + Sort output + Y + + + Sort output + Merge rows (diff) + Y + + + Merge rows (diff) + Not identical + Y + + + Not identical + Abort + Y + + + + Expected + DataGrid + + Y + + 1 + + none + + + + + N + -1 + id + -1 + Integer + + + N + -1 + name + -1 + String + + + + + 1 + alpha + + + 2 + bravo + + + 3 + charlie + + + 4 + delta + + + 5 + echo + + + + + 96 + 64 + + + + Output of the test + RowsFromResult + + Y + + 1 + + none + + + + + id + Integer + -1 + -1 + + + name + String + -1 + -1 + + + + + 96 + 224 + + + + Sort output + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + N + + + id + Y + N + N + 0 + N + + + name + Y + N + N + 0 + N + + + + + 256 + 224 + + + + Merge rows (diff) + MergeRows + + Y + + 1 + + none + + + + id + name + + + id + name + + flagfield + Expected + Sort output + + + 416 + 144 + + + + Not identical + FilterRows + + Y + + 1 + + none + + + + + + + N + flagfield + <> + + + constant + String + identical + -1 + -1 + N + + + + + + + 576 + 144 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0005-csv-filename-from-field: not every line of the named file reached the result on the single threaded engine + 0 + + + 736 + 144 + + + + + + diff --git a/integration-tests/single_threaded/0005-csv-filename-from-field.hpl b/integration-tests/single_threaded/0005-csv-filename-from-field.hpl new file mode 100644 index 00000000000..44a442dbdd1 --- /dev/null +++ b/integration-tests/single_threaded/0005-csv-filename-from-field.hpl @@ -0,0 +1,157 @@ + + + + + 0005-csv-filename-from-field + Y + CSV file input reads the file a row names on the single threaded engine: every line must reach the result (#9526) + + + Normal + + + N + 1000 + 100 + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + + file name + read the named file + Y + + + read the named file + Copy rows to result + Y + + + + file name + GetVariable + + Y + + 1 + + none + + + + + filename + ${PROJECT_HOME}/files/0005-names.csv + String + + + + + -1 + -1 + none + + + + + 96 + 96 + + + + read the named file + CSVInput + + Y + + 1 + + none + + + + filename + + N + ; + " +
Y
+ 50000 + N + N + N + N + + + + id + Integer + # + + + + -1 + -1 + none + + + name + String + + + + + -1 + -1 + none + + + + + 288 + 96 + +
+ + Copy rows to result + RowsToResult + + Y + + 1 + + none + + + + + 480 + 96 + + + + + +
diff --git a/integration-tests/single_threaded/0006-simple-mapping-with-input-child.hpl b/integration-tests/single_threaded/0006-simple-mapping-with-input-child.hpl new file mode 100644 index 00000000000..4ae9b02a16d --- /dev/null +++ b/integration-tests/single_threaded/0006-simple-mapping-with-input-child.hpl @@ -0,0 +1,98 @@ + + + + + 0006-simple-mapping-with-input-child + Y + The mapping of 0006-simple-mapping-with-input: passes its rows on + + + Normal + + + N + 1000 + 100 + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + + Input + Output + Y + + + + Input + MappingInput + + Y + + 1 + + none + + + + + + id + + Integer + + + + name + + String + + + + + 96 + 96 + + + + Output + MappingOutput + + Y + + 1 + + none + + + + + 288 + 96 + + + + + + diff --git a/integration-tests/single_threaded/0006-simple-mapping-with-input-validation.hpl b/integration-tests/single_threaded/0006-simple-mapping-with-input-validation.hpl new file mode 100644 index 00000000000..7fbbcb16619 --- /dev/null +++ b/integration-tests/single_threaded/0006-simple-mapping-with-input-validation.hpl @@ -0,0 +1,282 @@ + + + + + 0006-simple-mapping-with-input-validation + Y + Verify the result rows of 0006-simple-mapping-with-input, which ran on the single threaded engine + + + Normal + + + N + 1000 + 100 + - + 2026/11/06 12:00:00.000 + - + 2026/11/06 12:00:00.000 + + + + + + Expected + Merge rows (diff) + Y + + + Output of the test + Sort output + Y + + + Sort output + Merge rows (diff) + Y + + + Merge rows (diff) + Not identical + Y + + + Not identical + Abort + Y + + + + Expected + DataGrid + + Y + + 1 + + none + + + + + N + -1 + id + -1 + Integer + + + N + -1 + name + -1 + String + + + + + 1 + alpha + + + 2 + bravo + + + 3 + charlie + + + 4 + delta + + + 5 + echo + + + + + 96 + 64 + + + + Output of the test + RowsFromResult + + Y + + 1 + + none + + + + + id + Integer + -1 + -1 + + + name + String + -1 + -1 + + + + + 96 + 224 + + + + Sort output + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + N + + + id + Y + N + N + 0 + N + + + name + Y + N + N + 0 + N + + + + + 256 + 224 + + + + Merge rows (diff) + MergeRows + + Y + + 1 + + none + + + + id + name + + + id + name + + flagfield + Expected + Sort output + + + 416 + 144 + + + + Not identical + FilterRows + + Y + + 1 + + none + + + + + + + N + flagfield + <> + + + constant + String + identical + -1 + -1 + N + + + + + + + 576 + 144 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0006-simple-mapping-with-input: the rows of the Simple mapping on the single threaded engine are not the expected ones + 0 + + + 736 + 144 + + + + + + diff --git a/integration-tests/single_threaded/0006-simple-mapping-with-input.hpl b/integration-tests/single_threaded/0006-simple-mapping-with-input.hpl new file mode 100644 index 00000000000..6fdf3c4bc15 --- /dev/null +++ b/integration-tests/single_threaded/0006-simple-mapping-with-input.hpl @@ -0,0 +1,163 @@ + + + + + 0006-simple-mapping-with-input + Y + A Simple mapping with input on the single threaded engine: it must finish and give every row (#9526) + + + Normal + + + N + 1000 + 100 + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + + rows + Simple mapping + Y + + + Simple mapping + Copy rows to result + Y + + + + rows + DataGrid + + Y + + 1 + + none + + + + + N + -1 + id + -1 + Integer + + + N + -1 + name + -1 + String + + + + + 1 + alpha + + + 2 + bravo + + + 3 + charlie + + + 4 + delta + + + 5 + echo + + + + + 96 + 96 + + + + Simple mapping + SimpleMapping + + Y + + 1 + + none + + + ${PROJECT_HOME}/0006-simple-mapping-with-input-child.hpl + + + + Y + N + + + + + Y + N + + + + Y + + + + + + 288 + 96 + + + + Copy rows to result + RowsToResult + + Y + + 1 + + none + + + + + 480 + 96 + + + + + + diff --git a/integration-tests/single_threaded/files/0005-names.csv b/integration-tests/single_threaded/files/0005-names.csv new file mode 100644 index 00000000000..27d7349e8db --- /dev/null +++ b/integration-tests/single_threaded/files/0005-names.csv @@ -0,0 +1,6 @@ +id;name +1;alpha +2;bravo +3;charlie +4;delta +5;echo diff --git a/integration-tests/single_threaded/main-0005-csv-filename-from-field.hwf b/integration-tests/single_threaded/main-0005-csv-filename-from-field.hwf new file mode 100644 index 00000000000..68c43077591 --- /dev/null +++ b/integration-tests/single_threaded/main-0005-csv-filename-from-field.hwf @@ -0,0 +1,138 @@ + + + + main-0005-csv-filename-from-field + Y + + + + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + Start + + SPECIAL + + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 0 + 96 + 96 + + + + 0005-csv-filename-from-field.hpl + + PIPELINE + + ${PROJECT_HOME}/0005-csv-filename-from-field.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + single-threaded + + Y + + N + 0 + 320 + 96 + + + + 0005-csv-filename-from-field-validation.hpl + + PIPELINE + + ${PROJECT_HOME}/0005-csv-filename-from-field-validation.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 0 + 608 + 96 + + + + + + Start + 0005-csv-filename-from-field.hpl + 0 + 0 + Y + Y + Y + + + 0005-csv-filename-from-field.hpl + 0005-csv-filename-from-field-validation.hpl + 0 + 0 + Y + Y + N + + + + + + diff --git a/integration-tests/single_threaded/main-0006-simple-mapping-with-input.hwf b/integration-tests/single_threaded/main-0006-simple-mapping-with-input.hwf new file mode 100644 index 00000000000..a307b252691 --- /dev/null +++ b/integration-tests/single_threaded/main-0006-simple-mapping-with-input.hwf @@ -0,0 +1,138 @@ + + + + main-0006-simple-mapping-with-input + Y + + + + - + 2026/11/16 12:00:00.000 + - + 2026/11/16 12:00:00.000 + + + + + Start + + SPECIAL + + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 0 + 96 + 96 + + + + 0006-simple-mapping-with-input.hpl + + PIPELINE + + ${PROJECT_HOME}/0006-simple-mapping-with-input.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + single-threaded + + Y + + N + 0 + 320 + 96 + + + + 0006-simple-mapping-with-input-validation.hpl + + PIPELINE + + ${PROJECT_HOME}/0006-simple-mapping-with-input-validation.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 0 + 608 + 96 + + + + + + Start + 0006-simple-mapping-with-input.hpl + 0 + 0 + Y + Y + Y + + + 0006-simple-mapping-with-input.hpl + 0006-simple-mapping-with-input-validation.hpl + 0 + 0 + Y + Y + N + + + + + + diff --git a/plugins/transforms/mapping/src/main/java/org/apache/hop/pipeline/transforms/mapping/SimpleMapping.java b/plugins/transforms/mapping/src/main/java/org/apache/hop/pipeline/transforms/mapping/SimpleMapping.java index 62c103bc63d..f25b7fbc77c 100644 --- a/plugins/transforms/mapping/src/main/java/org/apache/hop/pipeline/transforms/mapping/SimpleMapping.java +++ b/plugins/transforms/mapping/src/main/java/org/apache/hop/pipeline/transforms/mapping/SimpleMapping.java @@ -127,7 +127,14 @@ public boolean processRow() throws HopException { if (data.rowDataInputMapper != null) { data.rowDataInputMapper.finished(); } - data.mappingPipeline.waitUntilFinished(); + if (data.executor != null) { + // A single-threaded mapping only runs when it is driven: nothing else finishes it + while (data.executor.oneIteration() && !isStopped()) { + // Keep driving iterations until all transforms in the sub-pipeline are done + } + } else { + data.mappingPipeline.waitUntilFinished(); + } setOutputDone(); return false; } diff --git a/plugins/transforms/mapping/src/test/java/org/apache/hop/pipeline/transforms/mapping/SimpleMappingTest.java b/plugins/transforms/mapping/src/test/java/org/apache/hop/pipeline/transforms/mapping/SimpleMappingTest.java index 8c0212283a1..601a4780481 100644 --- a/plugins/transforms/mapping/src/test/java/org/apache/hop/pipeline/transforms/mapping/SimpleMappingTest.java +++ b/plugins/transforms/mapping/src/test/java/org/apache/hop/pipeline/transforms/mapping/SimpleMappingTest.java @@ -336,6 +336,28 @@ void testProcessRowNoInputDrivesSingleThreadedLoop() throws HopException { verify(transformMockHelper.pipeline, times(1)).startThreads(); } + /** + * A single-threaded mapping with input only runs while it is driven. After the last input row + * nothing else finishes it, so waiting for it would wait forever. + */ + @Test + void testProcessRowEndOfInputDrivesSingleThreadedLoop() throws HopException { + when(transformMockHelper.iTransformMeta.isNoInput()).thenReturn(false); + when(transformMockHelper.iTransformMeta.isNoOutput()).thenReturn(true); + simpleMpData.mappingOutput = null; + SingleThreadedPipelineExecutor executor = mock(SingleThreadedPipelineExecutor.class); + when(executor.oneIteration()).thenReturn(true, true, false); + simpleMpData.executor = executor; + + smp = newSimpleMapping(); + smp.addRowSetToInputRowSets(transformMockHelper.getMockInputRowSet(new Object[] {})); + + assertTrue(smp.processRow(), "the input row goes to the mapping"); + assertFalse(smp.processRow(), "the end of the input finishes the mapping"); + verify(executor, times(3)).oneIteration(); + verify(transformMockHelper.pipeline, never()).waitUntilFinished(); + } + @Test void initRefusesIncomingHopWhenNoInputThroughTheEngine() { SimpleMapping mapping = mappingWithIncomingHop(); diff --git a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java index 6d55d328b62..b90dacde659 100644 --- a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java +++ b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java @@ -82,6 +82,11 @@ public boolean processRow() throws HopException { if (first && meta.isTruncateTable() && !meta.isOnlyWhenHaveRows()) { truncateTable(); } + // The end of the rows: write what is still buffered and say so, as processRow() must. A + // single-threaded run only disposes the transforms once all of them are done, so the + // transforms after this one would otherwise wait for more rows + emptyAndCommitBatchBuffers(false); + setOutputDone(); return false; } @@ -1450,7 +1455,12 @@ private void emptyAndCommitBatchBuffers(boolean dispose) { setErrors(1); stopAll(); } finally { - setOutputDone(); + // Only at the end of the run: batchComplete() flushes after every iteration of a + // single-threaded + // run, and the transforms after this one still read what follows + if (dispose) { + setOutputDone(); + } if (getErrors() > 0) { try { diff --git a/plugins/transforms/tableoutput/src/test/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputTest.java b/plugins/transforms/tableoutput/src/test/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputTest.java index 50385e6cbf3..f9378dc680a 100644 --- a/plugins/transforms/tableoutput/src/test/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputTest.java +++ b/plugins/transforms/tableoutput/src/test/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputTest.java @@ -32,6 +32,7 @@ import static org.mockito.Mockito.when; import java.sql.Connection; +import java.util.ArrayList; import java.util.Map; import org.apache.hop.core.database.Database; import org.apache.hop.core.database.DatabaseMeta; @@ -85,6 +86,9 @@ void setUp() { tableOutputData.tableName = "sas"; tableOutputData.preparedStatements = mock(Map.class); tableOutputData.commitCounterMap = mock(Map.class); + // The end of the rows flushes what is buffered: nothing here + tableOutputData.batchBuffer = new ArrayList<>(); + tableOutputData.batchBindBuffer = new ArrayList<>(); pipelineMeta = mock(PipelineMeta.class); doReturn(transformMeta).when(pipelineMeta).findTransform(anyString()); @@ -109,6 +113,32 @@ private void setupTableOutputSpy() { doNothing().when(tableOutputSpy).logDetailed(anyString()); } + /** + * A single-threaded run calls batchComplete() after every iteration. The transforms after this + * one still read the rows that follow, so only the end of the run closes the output: otherwise a + * transform reading an error hop or the output stops after the first quiet iteration. + */ + @Test + void flushingABatchLeavesTheOutputOpenUntilTheEndOfTheRun() throws Exception { + tableOutputSpy.batchComplete(); + verify(tableOutputSpy, never()).setOutputDone(); + + tableOutputSpy.dispose(); + verify(tableOutputSpy, times(1)).setOutputDone(); + } + + /** + * At the end of its rows the transform writes what it still holds and closes its output, as + * processRow() must: a single-threaded run disposes the transforms only once all are done. + */ + @Test + void theEndOfTheRowsClosesTheOutput() throws Exception { + doReturn(null).when(tableOutputSpy).getRow(); + + assertFalse(tableOutputSpy.processRow()); + verify(tableOutputSpy, times(1)).setOutputDone(); + } + @Test void testWriteToTable() throws Exception { tableOutputSpy.writeToTable(mock(IRowMeta.class), new Object[] {}); diff --git a/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputSingleThreadedTest.java b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputSingleThreadedTest.java new file mode 100644 index 00000000000..b1bc8c0a418 --- /dev/null +++ b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputSingleThreadedTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.csvinput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.plugins.TransformPluginType; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineHopMeta; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.SingleThreadedPipelineExecutor; +import org.apache.hop.pipeline.engines.local.LocalPipelineEngine; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.ITransform; +import org.apache.hop.pipeline.transform.ITransformData; +import org.apache.hop.pipeline.transform.ITransformMeta; +import org.apache.hop.pipeline.transform.RowAdapter; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transforms.dummy.DummyMeta; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * CSV file input reading the file a row names, run by the {@link SingleThreadedPipelineExecutor} + * for one iteration, as the single-threaded engine runs it. The transform reads one line per call, + * so it needs calls after its input has ended. + */ +class CsvInputSingleThreadedTest { + + private static final Duration TIMEOUT = Duration.ofSeconds(20); + + @TempDir Path folder; + + @BeforeAll + static void initHop() throws Exception { + HopEnvironment.init(); + PluginRegistry registry = PluginRegistry.getInstance(); + for (ITransformMeta meta : + List.of(new CsvInputMeta(), new DummyMeta(), new FileNameSourceMeta())) { + if (registry.getPluginId(TransformPluginType.class, meta) == null) { + registry.registerPluginClass( + meta.getClass().getName(), TransformPluginType.class, Transform.class); + } + } + } + + @Test + void theFileARowNamesIsReadToTheEnd() throws Exception { + Path file = folder.resolve("orders.csv"); + StringBuilder text = new StringBuilder("id;name\n"); + for (int id = 1; id <= 5; id++) { + text.append(id).append(";name ").append(id).append('\n'); + } + Files.writeString(file, text.toString()); + + List rows = new ArrayList<>(); + IRowMeta[] rowMeta = new IRowMeta[1]; + assertTimeoutPreemptively(TIMEOUT, () -> readSingleThreaded(file.toString(), rows, rowMeta)); + + assertEquals(5, rows.size()); + int id = rowMeta[0].indexOfValue("id"); + assertEquals(5L, rowMeta[0].getInteger(rows.get(4), id)); + } + + private static void readSingleThreaded(String filename, List rows, IRowMeta[] rowMeta) + throws Exception { + PipelineMeta pipelineMeta = new PipelineMeta(); + pipelineMeta.setName("csv-input-single-threaded"); + + CsvInputMeta csvInputMeta = new CsvInputMeta(); + csvInputMeta.setFilenameField("filename"); + csvInputMeta.setDelimiter(";"); + csvInputMeta.setHeaderPresent(true); + CsvInputField idField = new CsvInputField("id"); + idField.setType(IValueMeta.TYPE_INTEGER); + idField.setFormat("#"); + csvInputMeta.getInputFields().add(idField); + csvInputMeta.getInputFields().add(new CsvInputField("name")); + + FileNameSourceMeta fileNameSourceMeta = new FileNameSourceMeta(); + fileNameSourceMeta.filename = filename; + TransformMeta files = addTransform(pipelineMeta, "files", fileNameSourceMeta); + TransformMeta csv = addTransform(pipelineMeta, "csv", csvInputMeta); + TransformMeta output = addTransform(pipelineMeta, "output", new DummyMeta()); + pipelineMeta.addPipelineHop(new PipelineHopMeta(files, csv)); + pipelineMeta.addPipelineHop(new PipelineHopMeta(csv, output)); + + Pipeline pipeline = new LocalPipelineEngine(pipelineMeta); + pipeline.setPipelineType(PipelineMeta.PipelineType.SingleThreaded); + pipeline.prepareExecution(); + + ITransform csvTransform = pipeline.getTransform("csv", 0); + csvTransform.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(IRowMeta meta, Object[] row) { + rowMeta[0] = meta; + rows.add(row); + } + }); + + pipeline.startThreads(); + + SingleThreadedPipelineExecutor executor = new SingleThreadedPipelineExecutor(pipeline); + assertTrue(executor.init()); + try { + // One iteration, as the single-threaded engine runs + executor.oneIteration(); + } finally { + executor.dispose(); + } + assertEquals(0, csvTransform.getErrors()); + } + + private static TransformMeta addTransform( + PipelineMeta pipelineMeta, String name, ITransformMeta meta) { + String pluginId = PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta); + TransformMeta transformMeta = new TransformMeta(pluginId, name, meta); + pipelineMeta.addTransform(transformMeta); + return transformMeta; + } + + /** A source giving one row with a file name, all in its first call, as a Data grid does. */ + @Transform(id = "CsvInputSingleThreadedTestFileName", name = "File name source") + public static class FileNameSourceMeta + extends BaseTransformMeta { + String filename; + + @Override + public void getFields( + IRowMeta rowMeta, + String origin, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) { + rowMeta.addValueMeta(new ValueMetaString("filename")); + } + } + + public static class FileNameSourceData extends BaseTransformData implements ITransformData {} + + public static class FileNameSource extends BaseTransform { + public FileNameSource( + TransformMeta transformMeta, + FileNameSourceMeta meta, + FileNameSourceData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean processRow() throws HopException { + IRowMeta rowMeta = new RowMeta(); + meta.getFields(rowMeta, getTransformName(), null, null, this, null); + putRow(rowMeta, new Object[] {meta.filename}); + setOutputDone(); + return false; + } + } +}