diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
index 0dc52b19c9a..bf569b32e74 100644
--- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
+++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
@@ -31,9 +31,7 @@
import java.util.Objects;
import java.util.SortedMap;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hop.core.exception.HopException;
-import org.apache.hop.core.exception.HopRuntimeException;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.i18n.BaseMessages;
@@ -61,30 +59,121 @@ public DetectLanguage(
private final int cores = Runtime.getRuntime().availableProcessors();
private final int maxJobs = cores * 100;
private final ExecutorService executor = commonPool();
- private final AtomicInteger jobs = new AtomicInteger(0);
+
+ /** How long draining waits for accepted asynchronous work before abandoning it. */
+ private static final long DRAIN_TIMEOUT_MILLIS = 30_000L;
+
+ // Guarded by lock, including reservation before submission.
+ private int jobs;
+ private Throwable failure;
+ // Guards against surfacing the same asynchronous failure twice (finish/checkFailure and dispose).
+ private boolean failureReported;
private final Object lock = new Object();
private final LanguageDetector detector =
fromAllSpokenLanguages().withPreloadedLanguageModels().build();
- private void finish() {
- // Wait until all jobs are finished.
+ private void awaitJobs() {
+ awaitJobs(DRAIN_TIMEOUT_MILLIS);
+ }
+
+ private void awaitJobs(final long timeoutMillis) {
+ boolean interrupted = false;
+ final long deadline = System.currentTimeMillis() + timeoutMillis;
synchronized (lock) {
- while (jobs.get() > 0) {
+ long remaining = timeoutMillis;
+ while (jobs > 0 && remaining > 0) {
try {
- lock.wait();
+ lock.wait(remaining);
} catch (InterruptedException e) {
- throw new HopRuntimeException(e);
+ // Accepted work should finish before output is closed or resources are disposed, but an
+ // interrupted waiter must not hold up the transform or the pipeline.
+ interrupted = true;
}
+ remaining = deadline - System.currentTimeMillis();
+ }
+ if (jobs > 0) {
+ // A worker is stuck and no provider timeout applies. Do not block disposal forever: log and
+ // abandon the remaining jobs.
+ logBasic(
+ "Abandoning "
+ + jobs
+ + " asynchronous job(s) that did not finish within "
+ + timeoutMillis
+ + " ms");
}
}
- detector.unloadLanguageModels();
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Wait for every accepted job to complete on the normal end-of-input path. Unlike {@link
+ * #awaitJobs(long)} there is no timeout: rows that are merely slow must not be lost because the
+ * output is closed while workers still run. A stop (see {@link #stopRunning()}) breaks the wait.
+ */
+ private void awaitJobsUncapped() {
+ boolean interrupted = false;
+ synchronized (lock) {
+ while (jobs > 0 && !isStopped()) {
+ try {
+ lock.wait(1_000L);
+ } catch (InterruptedException e) {
+ // Accepted work should finish before output is closed, but an interrupted waiter must
+ // not hold up the transform or the pipeline.
+ interrupted = true;
+ }
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void checkFailure() throws HopException {
+ synchronized (lock) {
+ if (failure != null) {
+ failureReported = true;
+ throw new HopException("Asynchronous processing failed", failure);
+ }
+ }
+ }
+
+ private void finish() throws HopException {
+ awaitJobsUncapped();
+ checkFailure();
setOutputDone();
}
+ @Override
+ public void dispose() {
+ // The common pool is shared. Drain our work, but never shut the pool down.
+ awaitJobs();
+ detector.unloadLanguageModels();
+ final boolean shouldLog;
+ synchronized (lock) {
+ // Only log a failure we have not already surfaced through the run loop, so it is not logged
+ // twice; the error accounting below still applies in either case.
+ shouldLog = failure != null && !failureReported;
+ if (shouldLog) {
+ failureReported = true;
+ }
+ }
+ if (failure != null) {
+ if (shouldLog) {
+ logError("Asynchronous processing failed", failure);
+ }
+ setErrors(1);
+ stopAll();
+ }
+ super.dispose();
+ }
+
@Override
public boolean processRow() throws HopException {
+ checkFailure();
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) { // no more input to be expected...
finish();
@@ -108,12 +197,8 @@ public boolean processRow() throws HopException {
cacheIndexPositions();
} // End If first
- boolean sendToErrorRow = false;
- String errorMessage = null;
-
- process(r);
-
try {
+ process(r);
if (isRowLevel()) {
logRowlevel(
BaseMessages.getString(
@@ -122,27 +207,15 @@ public boolean processRow() throws HopException {
getLinesRead() + " : " + getInputRowMeta().getString(r)));
}
} catch (Exception e) {
- if (getTransformMeta().isDoingErrorHandling()) {
- sendToErrorRow = true;
- errorMessage = e.toString();
- } else {
- logError(
- BaseMessages.getString(PKG, "DetectLanguage.ErrorInTransformRunning") + e.getMessage());
- setErrors(1);
- stopAll();
- setOutputDone(); // signal end to receiver(s)
- return false;
- }
- if (sendToErrorRow) {
- // Simply add this row to the error row
- putError(getInputRowMeta(), r, 1, errorMessage, meta.getCorpusField(), "DetectLanguage001");
- }
+ // An infrastructure/worker failure must not be attributed to the current input row.
+ checkFailure();
+ handleRowError(r, e);
}
return true;
}
- private void process(Object[] inputRow) throws HopTransformException {
+ private void process(Object[] inputRow) throws HopException {
String corpus = trim((String) inputRow[data.indexOfCorpusField]);
if (isBlank(corpus)) {
return;
@@ -155,32 +228,62 @@ private void process(Object[] inputRow) throws HopTransformException {
}
}
- private void processAsync(String corpus, Object[] inputRow) {
+ private void handleRowError(Object[] row, Exception e) throws HopException {
+ if (!getTransformMeta().isDoingErrorHandling()) {
+ throw new HopException("Error processing row", e);
+ }
+ // Error output has the same serialization requirement as normal output.
+ synchronized (this) {
+ putError(getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(), "DetectLanguage001");
+ }
+ }
+
+ private void processAsync(String text, Object[] inputRow) throws HopException {
synchronized (lock) {
- while (jobs.get() > maxJobs) {
+ while (jobs >= maxJobs && failure == null) {
try {
lock.wait();
} catch (InterruptedException e) {
- throw new HopRuntimeException(e);
+ Thread.currentThread().interrupt();
+ throw new HopException("Interrupted waiting for async processing", e);
}
}
+ checkFailure();
+ jobs++;
}
- executor.submit(
- () -> {
- try {
- processSync(corpus, inputRow);
- jobs.decrementAndGet();
-
- synchronized (lock) {
- lock.notifyAll();
+ try {
+ executor.submit(
+ () -> {
+ try {
+ try {
+ processSync(text, inputRow);
+ } catch (Exception e) {
+ handleRowError(inputRow, e);
+ }
+ } catch (Throwable e) {
+ synchronized (lock) {
+ if (failure == null) {
+ failure = e;
+ }
+ }
+ } finally {
+ synchronized (lock) {
+ jobs--;
+ lock.notifyAll();
+ }
}
- } catch (Exception e) {
- logError(e.getMessage(), e);
- }
- });
-
- jobs.incrementAndGet();
+ });
+ } catch (RuntimeException | Error e) {
+ synchronized (lock) {
+ jobs--;
+ if (failure == null) {
+ failure = e;
+ }
+ lock.notifyAll();
+ }
+ checkFailure();
+ }
}
@SuppressWarnings(
diff --git a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
new file mode 100644
index 00000000000..3806cae0642
--- /dev/null
+++ b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
@@ -0,0 +1,434 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.language;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.github.pemistahl.lingua.api.LanguageDetector;
+import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
+import java.lang.reflect.Field;
+import java.util.List;
+import java.util.TreeMap;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.mockito.MockedStatic;
+
+@Timeout(10)
+class DetectLanguageAsyncTest {
+ private DetectLanguage transform;
+ private TransformMeta transformMeta;
+ private final Object[] row = {"
text
"};
+
+ @BeforeEach
+ void setUp() throws Exception {
+ org.apache.hop.core.logging.HopLogStore.init();
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.setCorpusField("input");
+ meta.setParallelism(true);
+ DetectLanguageData data = new DetectLanguageData();
+ data.outputRowMeta = new RowMeta();
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ transformMeta = spy(new TransformMeta("test", meta));
+ pipelineMeta.addTransform(transformMeta);
+ try (MockedStatic factory =
+ mockStatic(LanguageDetectorBuilder.class)) {
+ LanguageDetectorBuilder builder = mock(LanguageDetectorBuilder.class, RETURNS_SELF);
+ LanguageDetector detector = mock(LanguageDetector.class);
+ factory.when(LanguageDetectorBuilder::fromAllSpokenLanguages).thenReturn(builder);
+ when(builder.build()).thenReturn(detector);
+ when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new TreeMap<>());
+ transform = spy(new DetectLanguage(transformMeta, meta, data, 0, pipelineMeta, null));
+ }
+ transform.first = false;
+ data.indexOfCorpusField = 0;
+ data.indexOfDetectedLanguage = 1;
+ data.indexOfDetectedLanguageConfidence = 2;
+
+ doReturn(row, (Object[]) null).when(transform).getRow();
+ doNothing().when(transform).setOutputDone();
+ doNothing().when(transform).putRow(any(), any());
+ }
+
+ @Test
+ void inlineCompletionDoesNotLeavePhantomJob() throws Exception {
+ setField("executor", new InlineExecutor());
+ assertTrue(transform.processRow());
+ assertFalse(transform.processRow());
+ verify(transform).putRow(any(), any());
+ verify(transform).setOutputDone();
+ }
+
+ @Test
+ void workerExceptionIsVisibleAtEof() throws Exception {
+ setField("executor", new InlineExecutor());
+ doAnswer(
+ invocation -> {
+ throw new HopTransformException("worker failed");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ HopException failure = assertThrows(HopException.class, transform::processRow);
+ assertTrue(failure.getMessage().contains("Asynchronous processing failed"));
+ }
+
+ @Test
+ void workerErrorAlsoReleasesReservation() throws Exception {
+ setField("executor", new InlineExecutor());
+ doAnswer(
+ invocation -> {
+ throw new AssertionError("worker error");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertThrows(HopException.class, transform::processRow);
+ // Disposal must not hang even when submit captured an Error in its Future.
+ doNothing().when(transform).stopAll();
+ transform.dispose();
+ assertEquals(1, transform.getErrors());
+ }
+
+ @Test
+ void rejectedSubmissionRollsBackReservation() throws Exception {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ executor.shutdown();
+ setField("executor", executor);
+ assertThrows(HopException.class, transform::processRow);
+ doNothing().when(transform).stopAll();
+ transform.dispose();
+ assertEquals(1, transform.getErrors());
+ }
+
+ @Test
+ void synchronousFailureUsesConfiguredErrorHandling() throws Exception {
+ setParallelism(false);
+ doReturn(true).when(transformMeta).isDoingErrorHandling();
+ doAnswer(
+ invocation -> {
+ throw new HopTransformException("bad row");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ doNothing()
+ .when(transform)
+ .putError(
+ any(),
+ any(),
+ org.mockito.ArgumentMatchers.anyLong(),
+ anyString(),
+ anyString(),
+ anyString());
+ assertTrue(transform.processRow());
+ verify(transform)
+ .putError(
+ any(),
+ any(),
+ org.mockito.ArgumentMatchers.anyLong(),
+ anyString(),
+ anyString(),
+ anyString());
+ }
+
+ @Test
+ void asynchronousFailureUsesConfiguredErrorHandling() throws Exception {
+ setField("executor", new InlineExecutor());
+ doReturn(true).when(transformMeta).isDoingErrorHandling();
+ doAnswer(
+ invocation -> {
+ throw new HopTransformException("bad async row");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ doNothing()
+ .when(transform)
+ .putError(
+ any(),
+ any(),
+ org.mockito.ArgumentMatchers.anyLong(),
+ anyString(),
+ anyString(),
+ anyString());
+ assertTrue(transform.processRow());
+ assertFalse(transform.processRow());
+ verify(transform)
+ .putError(
+ any(),
+ any(),
+ org.mockito.ArgumentMatchers.anyLong(),
+ anyString(),
+ anyString(),
+ anyString());
+ }
+
+ @Test
+ void failureWakesAdmissionAtCapacity() throws Exception {
+ ExecutorService workers = Executors.newSingleThreadExecutor();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ setField("executor", workers);
+ setField("maxJobs", 1);
+ doReturn(row).when(transform).getRow();
+ doAnswer(
+ invocation -> {
+ entered.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ throw new HopTransformException("worker failed while admission waits");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ CountDownLatch admitting = new CountDownLatch(1);
+ Future> nextRow =
+ caller.submit(
+ () -> {
+ admitting.countDown();
+ return assertThrows(HopException.class, transform::processRow);
+ });
+ assertTrue(admitting.await(5, TimeUnit.SECONDS));
+ assertFalse(nextRow.isDone());
+ release.countDown();
+ nextRow.get(5, TimeUnit.SECONDS);
+ doNothing().when(transform).stopAll();
+ transform.dispose();
+ assertEquals(1, transform.getErrors());
+ } finally {
+ release.countDown();
+ workers.shutdownNow();
+ caller.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ void eofDrainsAcceptedWorkBeforeOutputDone() throws Exception {
+ assertDrains(false);
+ }
+
+ @Test
+ void disposalDrainsAcceptedWorkWithoutShuttingDownExecutor() throws Exception {
+ assertDrains(true);
+ }
+
+ private void assertDrains(boolean dispose) throws Exception {
+ ExecutorService workers = Executors.newSingleThreadExecutor();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicInteger outputs = new AtomicInteger();
+ try {
+ setField("executor", workers);
+ doAnswer(
+ invocation -> {
+ entered.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ outputs.incrementAndGet();
+ return null;
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ CountDownLatch draining = new CountDownLatch(1);
+ Future> completion =
+ caller.submit(
+ () -> {
+ draining.countDown();
+ if (dispose) {
+ transform.dispose();
+ } else {
+ try {
+ assertFalse(transform.processRow());
+ } catch (HopException e) {
+ throw new AssertionError(e);
+ }
+ }
+ });
+ assertTrue(draining.await(5, TimeUnit.SECONDS));
+ assertFalse(completion.isDone());
+ release.countDown();
+ completion.get(5, TimeUnit.SECONDS);
+ assertEquals(1, outputs.get());
+ assertFalse(workers.isShutdown());
+ if (!dispose) {
+ verify(transform).setOutputDone();
+ }
+ } finally {
+ release.countDown();
+ workers.shutdownNow();
+ caller.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ void eofWaitsForSlowInFlightJobAndDeliversRowBeforeOutputDone() throws Exception {
+ // Regression: the 30 s drain cap used to apply on the normal end-of-input path too, so finish()
+ // abandoned jobs still running after 30 s and dropped their rows. finish() must wait without a
+ // cap and only then close the output.
+ ExecutorService workers = Executors.newSingleThreadExecutor();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicInteger outputs = new AtomicInteger();
+ try {
+ setField("executor", workers);
+ doAnswer(
+ invocation -> {
+ entered.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ outputs.incrementAndGet();
+ return null;
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ // Run the EOF step on its own thread so we can observe that, while a job is still in flight,
+ // it keeps waiting instead of dropping the job and closing the output on a short drain cap.
+ CountDownLatch draining = new CountDownLatch(1);
+ Future> completion =
+ caller.submit(
+ () -> {
+ draining.countDown();
+ try {
+ assertFalse(transform.processRow());
+ } catch (HopException e) {
+ throw new AssertionError(e);
+ }
+ });
+ assertTrue(draining.await(5, TimeUnit.SECONDS));
+ // The in-flight job is still running; the EOF path must not have returned (closed the output)
+ // yet, even though it has been waiting longer than any sub-second drain cap.
+ Thread.sleep(100);
+ assertFalse(completion.isDone());
+ release.countDown();
+ completion.get(5, TimeUnit.SECONDS);
+ assertEquals(1, outputs.get());
+ verify(transform).setOutputDone();
+ assertFalse(workers.isShutdown());
+ } finally {
+ release.countDown();
+ workers.shutdownNow();
+ caller.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ void eofHonoursStopToShortCircuitUncappedWait() throws Exception {
+ // Regression: finish() waits without a cap for in-flight jobs, so a stop must still break the
+ // wait and close the output, otherwise a never-completing job would hang the transform.
+ ExecutorService workers = Executors.newSingleThreadExecutor();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ setField("executor", workers);
+ doAnswer(
+ invocation -> {
+ entered.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ return null;
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ doReturn(true).when(transform).isStopped();
+ assertFalse(transform.processRow());
+ verify(transform).setOutputDone();
+ } finally {
+ release.countDown();
+ workers.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ private void setParallelism(boolean parallelism) {
+ ((DetectLanguageMeta) transformMeta.getTransform()).setParallelism(parallelism);
+ }
+
+ private void setField(String name, Object value) throws Exception {
+ Field field = DetectLanguage.class.getDeclaredField(name);
+ field.setAccessible(true);
+ field.set(transform, value);
+ }
+
+ private static final class InlineExecutor extends AbstractExecutorService {
+ @Override
+ public void execute(Runnable command) {
+ command.run();
+ }
+
+ @Override
+ public void shutdown() {}
+
+ @Override
+ public List shutdownNow() {
+ return List.of();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return false;
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return false;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) {
+ return false;
+ }
+ }
+}
diff --git a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java
index a8590192563..a7325624c47 100644
--- a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java
+++ b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java
@@ -18,16 +18,23 @@
package org.apache.hop.pipeline.transforms.execprocess;
import java.io.BufferedReader;
+import java.io.Closeable;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.CountDownLatch;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
@@ -44,8 +51,13 @@ public class ExecProcess extends BaseTransform
private static final Class> PKG = ExecProcessMeta.class;
public static final String CONST_EXEC_PROCESS_EXCEPTION_COULD_NOT_FIND_FIELD =
"ExecProcess.Exception.CouldNotFindField";
- private boolean killing;
- private CountDownLatch waitForLatch;
+
+ /**
+ * How long a stopped process is given to exit on a graceful destroy before it is force-killed.
+ */
+ private static final long GRACE_PERIOD_SECONDS = 30;
+
+ private final AtomicReference activeExecution = new AtomicReference<>();
public ExecProcess(
TransformMeta transformMeta,
@@ -95,13 +107,19 @@ public boolean processRow() throws HopException {
cmdArray.add(Const.NVL(argString, ""));
}
- execProcess(cmdArray.toArray(new String[0]), processResult);
+ if (!execProcess(cmdArray.toArray(new String[0]), processResult)) {
+ setOutputDone();
+ return false;
+ }
} else {
if (isDetailed() && !hasBalancedQuotes(processString)) {
logDetailed(
BaseMessages.getString(PKG, "ExecProcess.Log.UnbalancedQuotes", processString));
}
- execProcess(tokenizeCommandLine(processString), processResult);
+ if (!execProcess(tokenizeCommandLine(processString), processResult)) {
+ setOutputDone();
+ return false;
+ }
}
if (meta.isFailWhenNotSuccess() && processResult.getExitValue() != 0) {
@@ -199,126 +217,198 @@ private void initializeAndLookup() throws HopException {
@Override
public void stopRunning() throws HopException {
- if (waitForLatch != null) {
- killing = true;
- try {
- waitForLatch.await();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new HopException("Interrupted exception while kill the process", e);
- }
+ // A pipeline stop must not wait for a child process or a stream reader. In particular, a
+ // descendant can keep a pipe open after the direct child exits.
+ setStopped(true);
+ final var execution = activeExecution.get();
+ if (execution != null) {
+ execution.cancelled.set(true);
}
}
- private void execProcess(String[] process, ProcessResult processresult) throws HopException {
- Process p = null;
- waitForLatch = new CountDownLatch(1);
+ private boolean execProcess(final String[] command, final ProcessResult result)
+ throws HopException {
+ final var execution = new ProcessExecution();
+ activeExecution.set(execution);
+ Process child = null;
+ boolean completed = false;
try {
- String errorMsg = null;
- // execute process
+ if (isStopped()) {
+ return false;
+ }
try {
- p = data.runtime.exec(process);
+ child = data.runtime.exec(command);
} catch (Exception e) {
- errorMsg = e.getMessage();
+ // An exec failure may be an IOException, SecurityException or IllegalArgumentException; all
+ // must reach the error hop rather than escape processRow as a non-HopException.
+ if (isStopped() || execution.cancelled.get()) {
+ return false;
+ }
+ result.setErrorStream(e.getMessage());
+ return true;
}
- if (p == null) {
- processresult.setErrorStream(errorMsg);
- } else {
- final Process child = p;
- // Drain stdout/stderr while the process runs. Reading only after onExit() can deadlock:
- // if the child fills either pipe buffer before exiting, it blocks on write forever.
- CompletableFuture stdoutFuture =
- CompletableFuture.supplyAsync(
- () -> {
- try {
- return getOutputString(
- new BufferedReader(new InputStreamReader(child.getInputStream())));
- } catch (IOException e) {
- throw new CompletionException(e);
- }
- });
- CompletableFuture stderrFuture =
- CompletableFuture.supplyAsync(
- () -> {
- try {
- return getOutputString(
- new BufferedReader(new InputStreamReader(child.getErrorStream())));
- } catch (IOException e) {
- throw new CompletionException(e);
- }
- });
-
- while (true) {
- try {
- if (child.waitFor(1, TimeUnit.SECONDS)) {
- break;
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new HopException(
- "Interrupted exception while running the process " + Arrays.toString(process) + "!",
- ie);
- }
- if (killing) {
- child.children().forEach(ProcessHandle::destroy);
- if (child.isAlive()) {
- child.destroy();
- }
- try {
- if (!child.waitFor(30, TimeUnit.SECONDS)) {
- child.destroyForcibly();
- child.waitFor();
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new HopException(
- "Interrupted exception while running the process "
- + Arrays.toString(process)
- + "!",
- ie);
- }
- logMinimal(BaseMessages.getString(PKG, "ExecProcess.AbortProcess"));
- break;
- }
+ if (isCancelled(execution)) {
+ return false;
+ }
+ final Process process = child;
+ // Owned daemon readers keep blocked pipes out of the shared common pool. They close their
+ // streams on normal completion; cancellation also arranges asynchronous stream cleanup.
+ final var stdout = execution.readers.submit(() -> readOutput(process.getInputStream()));
+ final var stderr = execution.readers.submit(() -> readOutput(process.getErrorStream()));
+ while (true) {
+ // Remember children while the parent is alive: after it exits, inherited pipes may
+ // still be held by descendants that are no longer discoverable through that parent.
+ try (final var descendants = process.descendants()) {
+ descendants.forEach(execution.descendants::add);
}
-
- try {
- processresult.setOutputStream(stdoutFuture.get());
- processresult.setErrorStream(stderrFuture.get());
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof IOException ioe) {
- throw ioe;
- }
- if (cause instanceof RuntimeException re) {
- throw re;
- }
- throw new IOException(cause);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new HopException(
- "Interrupted exception while running the process " + Arrays.toString(process) + "!",
- ie);
+ if (isCancelled(execution)) {
+ return false;
}
-
- processresult.setExitValue(child.exitValue());
+ // Poll at a second's granularity so we do not rescan /proc more often than needed.
+ if (process.waitFor(1, TimeUnit.SECONDS)) {
+ break;
+ }
+ }
+ final var output = awaitOutput(stdout, execution);
+ if (isCancelled(execution)) {
+ return false;
+ }
+ final var error = awaitOutput(stderr, execution);
+ if (isCancelled(execution)) {
+ return false;
}
- } catch (IOException ioe) {
- throw new HopException(
- "IO exception while running the process " + Arrays.toString(process) + "!", ioe);
- } catch (Exception e) {
- throw new HopException(e);
+ result.setOutputStream(output);
+ result.setErrorStream(error);
+ result.setExitValue(process.exitValue());
+ completed = true;
+ return true;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new HopException("Interrupted while running process " + Arrays.toString(command), e);
+ } catch (ExecutionException e) {
+ throw new HopException("Unable to read process output", e.getCause());
} finally {
- if (killing || waitForLatch.getCount() > 0) {
- waitForLatch.countDown();
- killing = false;
+ activeExecution.compareAndSet(execution, null);
+ execution.readers.shutdownNow();
+ if (child != null) {
+ if (completed) {
+ // Normal completion: the owned readers already closed the streams. Closing them again is
+ // a defensive no-op that needs no dedicated thread.
+ closeProcessStream(child.getOutputStream());
+ closeProcessStream(child.getInputStream());
+ closeProcessStream(child.getErrorStream());
+ } else {
+ // Aborted: destroy the process (graceful then forced) inline on the transform thread so a
+ // hop-run shutdown that exits the JVM cannot orphan a still-running command, then close
+ // its streams on a background thread.
+ cleanupProcess(child, execution);
+ }
}
- if (p != null) {
- p.destroy();
+ }
+ }
+
+ private boolean isCancelled(final ProcessExecution execution) {
+ return isStopped() || execution.cancelled.get();
+ }
+
+ private String awaitOutput(final Future output, final ProcessExecution execution)
+ throws InterruptedException, ExecutionException {
+ while (!isCancelled(execution)) {
+ try {
+ return output.get(100, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException e) {
+ // Observe stop requests even after the process exits but a stream drainer is still active.
+ }
+ }
+ return null;
+ }
+
+ private String readOutput(final InputStream stream) throws IOException {
+ try (final var reader = new BufferedReader(new InputStreamReader(stream))) {
+ return getOutputString(reader);
+ }
+ }
+
+ private void cleanupProcess(final Process process, final ProcessExecution execution) {
+ // Aborted: stop the process right away on the transform thread so a hop-run shutdown that calls
+ // System.exit() cannot orphan a still-running command. Only the stream close, that can block in
+ // native I/O or on a reader's monitor, is deferred to a background thread.
+ killProcess(process, execution);
+ closeStreamsAsync(process);
+ }
+
+ /**
+ * Stop the process gracefully (SIGTERM) and, after a grace period, force it (SIGKILL). This runs
+ * on the transform thread before processRow returns, so a pipeline shutdown that exits the JVM
+ * cannot orphan a still-running command. The daemon cleanup that used to own this work could be
+ * killed by System.exit() before the forced kill ever ran.
+ */
+ private void killProcess(final Process process, final ProcessExecution execution) {
+ try {
+ try (final var descendants = process.descendants()) {
+ descendants.forEach(execution.descendants::add);
}
+ // Ask gracefully first (SIGTERM) so a script trapping it can clean up and roll back, then
+ // fall back to a forced kill (SIGKILL) only if it does not exit.
+ execution.descendants.forEach(ProcessHandle::destroy);
+ if (process.isAlive()) {
+ process.destroy();
+ }
+ if (!process.waitFor(GRACE_PERIOD_SECONDS, TimeUnit.SECONDS)) {
+ execution.descendants.forEach(ProcessHandle::destroyForcibly);
+ if (process.isAlive()) {
+ process.destroyForcibly();
+ }
+ }
+ } catch (InterruptedException e) {
+ // A forced kill still makes sure the process goes away without waiting again.
+ Thread.currentThread().interrupt();
+ execution.descendants.forEach(ProcessHandle::destroyForcibly);
+ if (process.isAlive()) {
+ process.destroyForcibly();
+ }
+ }
+ }
+
+ /**
+ * Close the process streams on a daemon thread. stream.close() may block in native I/O or on a
+ * reader's monitor, so it must not run inline on the transform or pipeline thread.
+ */
+ private void closeStreamsAsync(final Process process) {
+ final var closer =
+ new Thread(
+ () -> {
+ closeProcessStream(process.getOutputStream());
+ closeProcessStream(process.getInputStream());
+ closeProcessStream(process.getErrorStream());
+ },
+ "hop-execprocess-cleanup");
+ closer.setDaemon(true);
+ closer.start();
+ }
+
+ private void closeProcessStream(final Closeable stream) {
+ try {
+ stream.close();
+ } catch (IOException e) {
+ logDebug("Unable to close process stream", e);
}
}
+ private static final class ProcessExecution {
+ private final AtomicBoolean cancelled = new AtomicBoolean();
+ // Owned by the executing (transform) thread, which also performs the graceful-then-forced kill.
+ private final Set descendants = new HashSet<>();
+ private final ExecutorService readers =
+ Executors.newFixedThreadPool(
+ 2,
+ task -> {
+ final var thread = new Thread(task, "hop-execprocess-output");
+ thread.setDaemon(true);
+ return thread;
+ });
+ }
+
/**
* Split a command line into the executable and its arguments. Tokens are separated by whitespace,
* but a section wrapped in single or double quotes is kept together as one token even when it
diff --git a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java
index 9aae436f365..bea7009d75e 100644
--- a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java
+++ b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java
@@ -23,10 +23,21 @@
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Stream;
import org.apache.hop.core.HopEnvironment;
import org.apache.hop.core.IRowSet;
import org.apache.hop.core.QueueRowSet;
@@ -41,6 +52,7 @@
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.condition.EnabledOnOs;
import org.junit.jupiter.api.condition.OS;
import org.junit.jupiter.api.extension.RegisterExtension;
@@ -359,6 +371,250 @@ void init_requiresResultFieldName() throws HopException {
assertFalse(transform.init());
}
+ @Test
+ @Timeout(10)
+ void stopWhileDrainingOutputDoesNotWaitForEndOfStream() throws Exception {
+ HopEnvironment.init();
+ final var transform =
+ newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"});
+ final var runtime = mock(Runtime.class);
+ final var process = mock(Process.class);
+ final var stdout = new BlockingInputStream();
+ when(process.getInputStream()).thenReturn(stdout);
+ when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+ when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream());
+ when(process.waitFor(1, TimeUnit.SECONDS)).thenReturn(true);
+ when(process.descendants()).thenAnswer(invocation -> Stream.empty());
+ when(runtime.exec(any(String[].class))).thenReturn(process);
+ transform.getData().runtime = runtime;
+ final var executor = Executors.newSingleThreadExecutor();
+ try {
+ final var running = executor.submit(transform::processRow);
+ assertTrue(stdout.reading.await(5, TimeUnit.SECONDS));
+ transform.stopRunning();
+ transform.stopRunning();
+ assertFalse(running.get(5, TimeUnit.SECONDS));
+ assertTrue(stdout.closed.await(5, TimeUnit.SECONDS));
+ assertEquals(0, transform.getErrors());
+ assertEquals(0, transform.getOutputRowSets().getFirst().size());
+ } finally {
+ stdout.close();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ @Timeout(10)
+ void stopDuringLaunchCancelsTheNewProcessWithoutBlockingStopper() throws Exception {
+ HopEnvironment.init();
+ final var transform =
+ newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"});
+ final var runtime = mock(Runtime.class);
+ final var process = mock(Process.class);
+ final var launching = new CountDownLatch(1);
+ final var releaseLaunch = new CountDownLatch(1);
+ final var destroyed = new CountDownLatch(1);
+ when(runtime.exec(any(String[].class)))
+ .thenAnswer(
+ invocation -> {
+ launching.countDown();
+ assertTrue(releaseLaunch.await(5, TimeUnit.SECONDS));
+ return process;
+ });
+ when(process.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+ when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+ when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream());
+ when(process.isAlive()).thenReturn(true);
+ when(process.descendants()).thenAnswer(invocation -> Stream.empty());
+ when(process.destroyForcibly())
+ .thenAnswer(
+ invocation -> {
+ destroyed.countDown();
+ return process;
+ });
+ transform.getData().runtime = runtime;
+ final var executor = Executors.newSingleThreadExecutor();
+ try {
+ final var running = executor.submit(transform::processRow);
+ assertTrue(launching.await(5, TimeUnit.SECONDS));
+ transform.stopRunning();
+ releaseLaunch.countDown();
+ assertFalse(running.get(5, TimeUnit.SECONDS));
+ assertTrue(destroyed.await(5, TimeUnit.SECONDS));
+ assertEquals(0, transform.getErrors());
+ } finally {
+ releaseLaunch.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ @Timeout(10)
+ void cancellationRetainsChildrenObservedBeforeParentExit() throws Exception {
+ HopEnvironment.init();
+ final var transform =
+ newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"});
+ final var runtime = mock(Runtime.class);
+ final var process = mock(Process.class);
+ final var descendant = mock(ProcessHandle.class);
+ final var destroyed = new CountDownLatch(1);
+ final var stdout = new BlockingInputStream();
+ when(descendant.destroyForcibly())
+ .thenAnswer(
+ invocation -> {
+ destroyed.countDown();
+ return true;
+ });
+ when(process.descendants()).thenReturn(Stream.of(descendant), Stream.empty());
+ when(process.getInputStream()).thenReturn(stdout);
+ when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+ when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream());
+ when(process.waitFor(1, TimeUnit.SECONDS)).thenReturn(true);
+ when(runtime.exec(any(String[].class))).thenReturn(process);
+ transform.getData().runtime = runtime;
+ final var executor = Executors.newSingleThreadExecutor();
+ try {
+ final var running = executor.submit(transform::processRow);
+ assertTrue(stdout.reading.await(5, TimeUnit.SECONDS));
+ transform.stopRunning();
+ assertFalse(running.get(5, TimeUnit.SECONDS));
+ assertTrue(destroyed.await(5, TimeUnit.SECONDS));
+ } finally {
+ stdout.close();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void stopBeforeLaunchDoesNotStartAProcess() throws Exception {
+ HopEnvironment.init();
+ final var transform =
+ newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"});
+ final var runtime = mock(Runtime.class);
+ transform.getData().runtime = runtime;
+ transform.stopRunning();
+ assertFalse(transform.processRow());
+ verify(runtime, never()).exec(any(String[].class));
+ }
+
+ @Test
+ @Timeout(10)
+ void stoppedProcessThatExitsWithinGracePeriodIsDestroyedNotForceKilled() throws Exception {
+ HopEnvironment.init();
+ final var transform =
+ newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"});
+ final var runtime = mock(Runtime.class);
+ final var process = mock(Process.class);
+ final var stdout = new BlockingInputStream();
+ when(process.getInputStream()).thenReturn(stdout);
+ when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+ when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream());
+ when(process.isAlive()).thenReturn(true);
+ when(process.descendants()).thenAnswer(invocation -> Stream.empty());
+ // The process has not exited on its own while it is polled ...
+ when(process.waitFor(1, TimeUnit.SECONDS)).thenReturn(false);
+ // ... but a graceful destroy makes it exit within the grace period.
+ when(process.waitFor(30, TimeUnit.SECONDS)).thenReturn(true);
+ when(runtime.exec(any(String[].class))).thenReturn(process);
+ transform.getData().runtime = runtime;
+ final var executor = Executors.newSingleThreadExecutor();
+ try {
+ final var running = executor.submit(transform::processRow);
+ assertTrue(stdout.reading.await(5, TimeUnit.SECONDS));
+ transform.stopRunning();
+ assertFalse(running.get(5, TimeUnit.SECONDS));
+ // A graceful destroy must be tried first, and a forced kill never becomes necessary when the
+ // process cooperates within the grace period.
+ assertTrue(stdout.closed.await(5, TimeUnit.SECONDS));
+ verify(process).destroy();
+ verify(process, never()).destroyForcibly();
+ } finally {
+ stdout.close();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ @Timeout(10)
+ void forceKillRunsInlineOnTransformThreadBeforeProcessRowReturns() throws Exception {
+ // Regression: the forced kill used to run on a daemon cleanup thread that hop-run's
+ // System.exit() could kill before destroyForcibly() ever ran, leaving a SIGTERM-ignoring
+ // command running. The graceful-then-forced kill must now run inline on the transform thread so
+ // processRow() does not return until the process has been forcibly killed.
+ HopEnvironment.init();
+ final var transform =
+ newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"});
+ final var runtime = mock(Runtime.class);
+ final var process = mock(Process.class);
+ final var stdout = new BlockingInputStream();
+ final var forceKillEntered = new CountDownLatch(1);
+ final var releaseForceKill = new CountDownLatch(1);
+ when(process.getInputStream()).thenReturn(stdout);
+ when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+ when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream());
+ when(process.isAlive()).thenReturn(true);
+ when(process.descendants()).thenAnswer(invocation -> Stream.empty());
+ // The process never exits on its own, neither in the poll loop nor within the grace period.
+ when(process.waitFor(1, TimeUnit.SECONDS)).thenReturn(false);
+ when(process.waitFor(30, TimeUnit.SECONDS)).thenReturn(false);
+ when(process.destroyForcibly())
+ .thenAnswer(
+ invocation -> {
+ forceKillEntered.countDown();
+ assertTrue(releaseForceKill.await(5, TimeUnit.SECONDS));
+ return process;
+ });
+ when(runtime.exec(any(String[].class))).thenReturn(process);
+ transform.getData().runtime = runtime;
+ final var executor = Executors.newSingleThreadExecutor();
+ try {
+ final var running = executor.submit(transform::processRow);
+ assertTrue(stdout.reading.await(5, TimeUnit.SECONDS));
+ transform.stopRunning();
+ assertTrue(forceKillEntered.await(5, TimeUnit.SECONDS));
+ // processRow is still running because the forced kill holds the transform thread; it must not
+ // have returned and left the kill to a background thread that System.exit() could kill.
+ assertThrows(TimeoutException.class, () -> running.get(2, TimeUnit.SECONDS));
+ releaseForceKill.countDown();
+ assertFalse(running.get(5, TimeUnit.SECONDS));
+ verify(process).destroy();
+ verify(process).destroyForcibly();
+ assertEquals(0, transform.getErrors());
+ } finally {
+ releaseForceKill.countDown();
+ stdout.close();
+ executor.shutdownNow();
+ }
+ }
+
+ private static final class BlockingInputStream extends InputStream {
+ private final CountDownLatch reading = new CountDownLatch(1);
+ private final CountDownLatch closed = new CountDownLatch(1);
+
+ @Override
+ public int read() {
+ reading.countDown();
+ boolean interrupted = false;
+ while (true) {
+ try {
+ closed.await();
+ break;
+ } catch (InterruptedException e) {
+ interrupted = true;
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ return -1;
+ }
+
+ @Override
+ public void close() {
+ closed.countDown();
+ }
+ }
+
private ExecProcess newTransform(ExecProcessMeta meta, IRowMeta rowMeta, Object[] row) {
TransformMeta tm = smh.transformMeta;
when(tm.isDoingErrorHandling()).thenReturn(false);
diff --git a/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java b/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java
index 0adaa88d9b5..82736e3ebd7 100644
--- a/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java
+++ b/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java
@@ -17,13 +17,11 @@
package org.apache.hop.pipeline.transforms.languagemodelchat;
-import static java.lang.Runtime.getRuntime;
import static java.lang.System.arraycopy;
import static java.time.Duration.between;
import static java.time.Instant.now;
import static java.util.concurrent.Executors.newWorkStealingPool;
import static java.util.concurrent.ForkJoinPool.commonPool;
-import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.commons.lang3.StringUtils.trim;
@@ -37,7 +35,7 @@
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopRuntimeException;
@@ -62,11 +60,15 @@ public class LanguageModelChat extends BaseTransform facadeMap = new ConcurrentHashMap<>();
private LanguageModelChatMeta modelMeta;
- private final int parallelism = 1;
- private ForkJoinPool executor;
+ private volatile ExecutorService executor;
+ private volatile boolean ownsExecutor;
private final int cores = Runtime.getRuntime().availableProcessors();
private final AtomicInteger jobs = new AtomicInteger(0);
private final int maxJobs = cores * 100;
+
+ /** How long draining waits for accepted asynchronous work before abandoning it. */
+ private static final long DRAIN_TIMEOUT_MILLIS = 30_000L;
+
private final Object lock = new Object();
public LanguageModelChat(
@@ -80,29 +82,68 @@ public LanguageModelChat(
}
private void finish() {
+ awaitJobsUncapped();
+ shutdownOwnedExecutor();
+
+ setOutputDone();
+ }
+
+ private void awaitJobs() {
+ awaitJobs(DRAIN_TIMEOUT_MILLIS);
+ }
+ private void awaitJobs(long timeoutMillis) {
+ boolean interrupted = false;
+ final long deadline = System.currentTimeMillis() + timeoutMillis;
synchronized (lock) {
- while (jobs.get() > 0) {
+ long remaining = timeoutMillis;
+ while (jobs.get() > 0 && remaining > 0) {
try {
- lock.wait();
+ lock.wait(remaining);
} catch (InterruptedException e) {
- throw new HopRuntimeException(e);
+ // Accepted work should finish before output is closed or resources are disposed, but an
+ // interrupted waiter must not hold up the transform or the pipeline forever.
+ interrupted = true;
}
+ remaining = deadline - System.currentTimeMillis();
}
+ if (jobs.get() > 0) {
+ // A worker is stuck and no provider timeout applies. Do not block disposal forever: log and
+ // abandon the remaining jobs.
+ logBasic(
+ "Abandoning "
+ + jobs.get()
+ + " asynchronous job(s) that did not finish within "
+ + timeoutMillis
+ + " ms");
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
}
+ }
- if (executor != null) {
- try {
- while (executor.hasQueuedSubmissions() || executor.getActiveThreadCount() > 0) {
- SECONDS.sleep(5);
+ /**
+ * Wait for every accepted job to complete on the normal end-of-input path. Unlike {@link
+ * #awaitJobs(long)} there is no timeout: rows that are merely slow must not be lost because the
+ * output is closed (and the facades dropped) while workers still run. A stop breaks the wait.
+ */
+ private void awaitJobsUncapped() {
+ boolean interrupted = false;
+ synchronized (lock) {
+ while (jobs.get() > 0 && !isStopped()) {
+ try {
+ lock.wait(1_000L);
+ } catch (InterruptedException e) {
+ // Accepted work should finish before output is closed, but an interrupted waiter must
+ // not hold up the transform or the pipeline.
+ interrupted = true;
}
- } catch (InterruptedException e) {
- logError(e.getMessage(), e);
}
}
-
- facadeMap.clear();
- setOutputDone();
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
}
@Override
@@ -127,14 +168,11 @@ public boolean processRow() throws HopException {
new LanguageModel(modelMeta),
new LanguageModelFacade(variables, modelMeta, getLogChannel()));
- int parallelism =
- meta.getParallelism() <= 0 ? getRuntime().availableProcessors() : meta.getParallelism();
- if (parallelism != 0 && this.parallelism != parallelism || executor == null) {
- if (executor != null) {
- executor.shutdownNow();
- }
- executor = (ForkJoinPool) newWorkStealingPool(parallelism);
- }
+ ownsExecutor = meta.getParallelism() != 0;
+ executor =
+ ownsExecutor
+ ? newWorkStealingPool(meta.getParallelism() < 0 ? cores : meta.getParallelism())
+ : commonPool();
if (isEmpty(meta.getInputField())) {
throw new HopException(
@@ -179,53 +217,93 @@ private void process(Object[] inputRow) {
}
synchronized (lock) {
- while (jobs.get() > maxJobs) {
+ while (jobs.get() >= maxJobs && !isStopped()) {
try {
lock.wait();
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new HopRuntimeException(e);
}
}
}
- ExecutorService exec = meta.getParallelism() == 0 ? commonPool() : executor;
-
- exec.submit(
- () -> {
- try {
- processSync(inputRow, message);
- } catch (Exception e) {
+ if (isStopped()) {
+ return;
+ }
+ jobs.incrementAndGet();
+ try {
+ executor.submit(
+ () -> {
try {
- if (getTransformMeta().isDoingErrorHandling()) {
- String errorMessage = e.toString();
- putError(
- getInputRowMeta(),
- inputRow,
- 1L,
- errorMessage,
- meta.getInputField(),
- "LanguageModelChat001");
- } else {
- logError(i18n("LanguageModelChat.ErrorInTransformRunning") + e.getMessage(), e);
- setErrors(1L);
- stopAll();
+ processSync(inputRow, message);
+ } catch (Exception e) {
+ try {
+ if (getTransformMeta().isDoingErrorHandling()) {
+ String errorMessage = e.toString();
+ putError(
+ getInputRowMeta(),
+ inputRow,
+ 1L,
+ errorMessage,
+ meta.getInputField(),
+ "LanguageModelChat001");
+ } else {
+ logError(i18n("LanguageModelChat.ErrorInTransformRunning") + e.getMessage(), e);
+ setErrors(1L);
+ stopAll();
+ }
+ } catch (HopTransformException ex) {
+ logError(e.getMessage(), e);
+ logError(ex.getMessage(), ex);
+ if (!getTransformMeta().isDoingErrorHandling()) {
+ setErrors(1L);
+ stopAll();
+ }
}
- } catch (HopTransformException ex) {
- logError(e.getMessage(), e);
- logError(ex.getMessage(), ex);
- if (!getTransformMeta().isDoingErrorHandling()) {
- setErrors(1L);
- stopAll();
- }
- }
- } finally {
- jobs.decrementAndGet();
- synchronized (lock) {
- lock.notifyAll();
+ } finally {
+ jobFinished();
}
- }
- });
- jobs.incrementAndGet();
+ });
+ } catch (RejectedExecutionException e) {
+ jobFinished();
+ if (!isStopped()) {
+ throw e;
+ }
+ }
+ }
+
+ private void jobFinished() {
+ jobs.decrementAndGet();
+ synchronized (lock) {
+ lock.notifyAll();
+ }
+ }
+
+ private void shutdownOwnedExecutor() {
+ if (ownsExecutor && executor != null) {
+ // Accepted tasks still drain. Never await this executor from a worker stopping the pipeline.
+ executor.shutdown();
+ }
+ }
+
+ @Override
+ public void stopRunning() throws HopException {
+ setStopped(true);
+ shutdownOwnedExecutor();
+ synchronized (lock) {
+ lock.notifyAll();
+ }
+ super.stopRunning();
+ }
+
+ @Override
+ public void dispose() {
+ shutdownOwnedExecutor();
+ awaitJobs();
+ // The runner disposes after processRow returns, which need not have reached normal EOF after
+ // a stop or failure. Keep facades alive until every accepted worker has left processSync.
+ facadeMap.clear();
+ super.dispose();
}
private void processSync(Object[] inputRow, String message)
diff --git a/plugins/transforms/languagemodelchat/src/test/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChatLifecycleTest.java b/plugins/transforms/languagemodelchat/src/test/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChatLifecycleTest.java
new file mode 100644
index 00000000000..6db2c2f69ad
--- /dev/null
+++ b/plugins/transforms/languagemodelchat/src/test/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChatLifecycleTest.java
@@ -0,0 +1,354 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.languagemodelchat;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockConstruction;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
+
+import dev.langchain4j.data.message.AiMessage;
+import dev.langchain4j.model.chat.response.ChatResponse;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinWorkerThread;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.QueueRowSet;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transforms.languagemodelchat.internals.LanguageModelFacade;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@Timeout(15)
+class LanguageModelChatLifecycleTest {
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension();
+
+ private TransformMockHelper helper;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ helper =
+ new TransformMockHelper<>("chat", LanguageModelChatMeta.class, LanguageModelChatData.class);
+ when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(helper.iLogChannel);
+ when(helper.pipeline.isRunning()).thenReturn(true);
+ HopEnvironment.init();
+ }
+
+ @AfterEach
+ void cleanUp() {
+ helper.cleanUp();
+ }
+
+ private LanguageModelChat transform(final int parallelism) {
+ final var meta = new LanguageModelChatMeta();
+ meta.setDefault();
+ meta.setMock(true);
+ meta.setMockOutputValue("answer");
+ meta.setParallelism(parallelism);
+ final var transform =
+ new LanguageModelChat(
+ helper.transformMeta,
+ meta,
+ new LanguageModelChatData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+ final var rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("input"));
+ transform.setInputRowMeta(rowMeta);
+ final var input = helper.getMockInputRowSet(new Object[] {"question"});
+ when(input.getRowMeta()).thenReturn(rowMeta);
+ transform.addRowSetToInputRowSets(input);
+ transform.addRowSetToOutputRowSets(new QueueRowSet());
+ return transform;
+ }
+
+ @Test
+ void ownedPoolDrainsMockOutputBeforeCompletion() throws Exception {
+ final var transform = transform(2);
+ final var pool = new AtomicReference();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(final IRowMeta rowMeta, final Object[] row) {
+ pool.set(((ForkJoinWorkerThread) Thread.currentThread()).getPool());
+ }
+ });
+ assertTrue(transform.processRow());
+ assertFalse(transform.processRow());
+ final var output = transform.getOutputRowSets().getFirst();
+ assertEquals(1, output.size());
+ final var row = output.getRow();
+ assertEquals("MOCK", row[transform.getData().indexOfFinishReason]);
+ assertEquals("answer", row[transform.getData().indexOfOutput]);
+ assertTrue(output.isDone());
+ transform.dispose();
+ transform.dispose();
+ assertTrue(pool.get().isShutdown());
+ assertTrue(pool.get().awaitTermination(5, TimeUnit.SECONDS));
+ assertEquals(0, transform.getErrors());
+ }
+
+ @Test
+ void providerExceptionStillProducesAnErrorResultRow() throws Exception {
+ try (final var facades =
+ mockConstruction(
+ LanguageModelFacade.class,
+ (facade, context) -> {
+ when(facade.inputToChatMessages(any())).thenReturn(List.of());
+ when(facade.chat(any())).thenThrow(new IllegalStateException("provider unavailable"));
+ })) {
+ final var transform = transform(1);
+ transform.getMeta().setMock(false);
+ assertTrue(transform.processRow());
+ assertFalse(transform.processRow());
+ final var row = transform.getOutputRowSets().getFirst().getRow();
+ assertEquals("ERROR", row[transform.getData().indexOfFinishReason]);
+ assertEquals("provider unavailable", row[transform.getData().indexOfOutput]);
+ assertEquals(0, transform.getErrors());
+ transform.dispose();
+ }
+ }
+
+ @Test
+ void rejectionDuringStopIsNotReportedAsARowFailure() throws Exception {
+ final var executor = mock(ExecutorService.class);
+ try (final var factory = mockStatic(Executors.class)) {
+ factory.when(() -> Executors.newWorkStealingPool(1)).thenReturn(executor);
+ final var transform = transform(1);
+ when(executor.submit(any(Runnable.class)))
+ .thenAnswer(
+ invocation -> {
+ transform.stopRunning();
+ throw new RejectedExecutionException("stopping");
+ });
+ assertTrue(transform.processRow());
+ transform.dispose();
+ assertEquals(0, transform.getErrors());
+ }
+ }
+
+ @Test
+ void commonPoolRemainsUsableAfterCompletionAndDisposal() throws Exception {
+ final var transform = transform(0);
+ assertTrue(transform.processRow());
+ assertFalse(transform.processRow());
+ transform.dispose();
+ transform.stopRunning();
+ assertFalse(ForkJoinPool.commonPool().isShutdown());
+ assertEquals(42, ForkJoinPool.commonPool().submit(() -> 42).get(5, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void workerFailureCanStopThePipelineWithoutWaitingForItself() throws Exception {
+ final var transform = transform(1);
+ final var stopping = new CountDownLatch(1);
+ doAnswer(
+ invocation -> {
+ transform.stopRunning();
+ stopping.countDown();
+ return null;
+ })
+ .when(helper.pipeline)
+ .stopAll();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(final IRowMeta rowMeta, final Object[] row)
+ throws HopTransformException {
+ throw new HopTransformException("output failed");
+ }
+ });
+ assertTrue(transform.processRow());
+ assertTrue(stopping.await(5, TimeUnit.SECONDS));
+ transform.dispose();
+ assertEquals(1, transform.getErrors());
+ }
+
+ @Test
+ void disposeWaitsForAcceptedOutputWithoutHoldingItsMonitor() throws Exception {
+ final var transform = transform(1);
+ final var outputStarted = new CountDownLatch(1);
+ final var releaseOutput = new CountDownLatch(1);
+ final var workerFinished = new AtomicBoolean();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(final IRowMeta rowMeta, final Object[] row)
+ throws HopTransformException {
+ outputStarted.countDown();
+ try {
+ if (!releaseOutput.await(5, TimeUnit.SECONDS)) {
+ throw new HopTransformException("output wait timed out");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new HopTransformException(e);
+ }
+ workerFinished.set(true);
+ }
+ });
+ final var disposer = Executors.newSingleThreadExecutor();
+ try {
+ assertTrue(transform.processRow());
+ assertTrue(outputStarted.await(5, TimeUnit.SECONDS));
+ transform.stopRunning();
+ final var disposed =
+ disposer.submit(
+ () -> {
+ transform.dispose();
+ assertTrue(
+ workerFinished.get(), "dispose returned before accepted output completed");
+ });
+ assertFalse(disposed.isDone());
+ releaseOutput.countDown();
+ disposed.get(5, TimeUnit.SECONDS);
+ assertEquals(0, transform.getErrors());
+ } finally {
+ releaseOutput.countDown();
+ disposer.shutdownNow();
+ }
+ }
+
+ @Test
+ void queuedWorkAfterStopRetainsFacadeAndDrainsOnDispose() throws Exception {
+ final var executor = Executors.newSingleThreadExecutor();
+ final var queueBlocked = new CountDownLatch(1);
+ final var releaseQueue = new CountDownLatch(1);
+ final var workCompleted = new AtomicBoolean();
+ final var blocker =
+ executor.submit(
+ () -> {
+ queueBlocked.countDown();
+ assertTrue(releaseQueue.await(5, TimeUnit.SECONDS));
+ return null;
+ });
+ try (final var facades =
+ mockConstruction(
+ LanguageModelFacade.class,
+ (facade, context) ->
+ when(facade.inputToChatMessages(any()))
+ .thenAnswer(
+ invocation -> {
+ workCompleted.set(true);
+ return List.of();
+ }));
+ final var factory = mockStatic(Executors.class, CALLS_REAL_METHODS)) {
+ assertTrue(queueBlocked.await(5, TimeUnit.SECONDS));
+ factory.when(() -> Executors.newWorkStealingPool(1)).thenReturn(executor);
+ final var transform = transform(1);
+ assertTrue(transform.processRow());
+ transform.stopRunning();
+ assertFalse(transform.processRow());
+ assertFalse(workCompleted.get());
+
+ releaseQueue.countDown();
+ transform.dispose();
+ assertTrue(workCompleted.get(), "dispose must drain accepted queued work after stop");
+ assertEquals(0, transform.getErrors());
+ blocker.get(5, TimeUnit.SECONDS);
+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ } finally {
+ releaseQueue.countDown();
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ void finishWaitsForSlowChatOutputBeforeClosingOutput() throws Exception {
+ // Regression: the 30 s drain cap used to apply on the normal end-of-input path too, so finish()
+ // abandoned a slow in-flight chat, cleared the facades and closed the output, dropping the row.
+ // finish() must wait without a cap so the row is still delivered.
+ final var chatStarted = new CountDownLatch(1);
+ final var releaseChat = new CountDownLatch(1);
+ try (final var facades =
+ mockConstruction(
+ LanguageModelFacade.class,
+ (facade, context) -> {
+ final var aiMessage = mock(AiMessage.class);
+ when(aiMessage.text()).thenReturn("answer");
+ final var aiResponse = mock(ChatResponse.class);
+ when(aiResponse.aiMessage()).thenReturn(aiMessage);
+ when(facade.inputToChatMessages(any())).thenReturn(List.of());
+ when(facade.chat(any()))
+ .thenAnswer(
+ invocation -> {
+ chatStarted.countDown();
+ assertTrue(releaseChat.await(5, TimeUnit.SECONDS));
+ return aiResponse;
+ });
+ })) {
+ final var transform = transform(1);
+ transform.getMeta().setMock(false);
+ assertTrue(transform.processRow());
+ assertTrue(chatStarted.await(5, TimeUnit.SECONDS));
+ final var eof = Executors.newSingleThreadExecutor();
+ try {
+ final var completion =
+ eof.submit(
+ () -> {
+ try {
+ assertFalse(transform.processRow());
+ } catch (HopException e) {
+ throw new AssertionError(e);
+ }
+ });
+ // The end-of-input path must keep waiting for the in-flight chat rather than abandoning it
+ // after a short drain cap and closing the output.
+ Thread.sleep(100);
+ assertFalse(completion.isDone());
+ releaseChat.countDown();
+ completion.get(5, TimeUnit.SECONDS);
+ final var row = transform.getOutputRowSets().getFirst().getRow();
+ assertEquals("answer", row[transform.getData().indexOfOutput]);
+ assertEquals(0, transform.getErrors());
+ transform.dispose();
+ } finally {
+ releaseChat.countDown();
+ eof.shutdownNow();
+ }
+ }
+ }
+}
diff --git a/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java b/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java
index 3ee791ab522..6952ae93619 100644
--- a/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java
+++ b/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java
@@ -26,8 +26,7 @@
import static org.apache.hop.pipeline.transforms.stanford.nlp.simple.ParsedSentence.Builder.parsedSentenceBuilder;
import java.util.List;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.ExecutorService;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopRuntimeException;
import org.apache.hop.core.exception.HopTransformException;
@@ -44,8 +43,16 @@ public class StanfordSimpleNlp extends BaseTransform PKG = StanfordSimpleNlp.class; // For Translator
private final int cores = Runtime.getRuntime().availableProcessors();
- private final ForkJoinPool executor = commonPool();
- private final AtomicInteger jobs = new AtomicInteger(0);
+ private final ExecutorService executor = commonPool();
+
+ /** How long draining waits for accepted asynchronous work before abandoning it. */
+ private static final long DRAIN_TIMEOUT_MILLIS = 30_000L;
+
+ // Guarded by lock, including reservation before submission.
+ private int jobs;
+ private Throwable failure;
+ // Guards against surfacing the same asynchronous failure twice (finish/checkFailure and dispose).
+ private boolean failureReported;
private final int maxJobs = cores * 100;
private final Object lock = new Object();
private ParsedSentence.Builder sentenceBuilder;
@@ -60,22 +67,105 @@ public StanfordSimpleNlp(
super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
}
- private void finish() {
- // Wait until all jobs are finished.
+ private void awaitJobs() {
+ awaitJobs(DRAIN_TIMEOUT_MILLIS);
+ }
+
+ private void awaitJobs(final long timeoutMillis) {
+ boolean interrupted = false;
+ final long deadline = System.currentTimeMillis() + timeoutMillis;
synchronized (lock) {
- while (jobs.get() > 0) {
+ long remaining = timeoutMillis;
+ while (jobs > 0 && remaining > 0) {
try {
- lock.wait();
+ lock.wait(remaining);
} catch (InterruptedException e) {
- throw new HopRuntimeException(e);
+ // Accepted work should finish before output is closed or resources are disposed, but an
+ // interrupted waiter must not hold up the transform or the pipeline.
+ interrupted = true;
}
+ remaining = deadline - System.currentTimeMillis();
+ }
+ if (jobs > 0) {
+ // A worker is stuck and no provider timeout applies. Do not block disposal forever: log and
+ // abandon the remaining jobs.
+ logBasic(
+ "Abandoning "
+ + jobs
+ + " asynchronous job(s) that did not finish within "
+ + timeoutMillis
+ + " ms");
}
}
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Wait for every accepted job to complete on the normal end-of-input path. Unlike {@link
+ * #awaitJobs(long)} there is no timeout: rows that are merely slow must not be lost because the
+ * output is closed while workers still run. A stop (see {@link #stopRunning()}) breaks the wait.
+ */
+ private void awaitJobsUncapped() {
+ boolean interrupted = false;
+ synchronized (lock) {
+ while (jobs > 0 && !isStopped()) {
+ try {
+ lock.wait(1_000L);
+ } catch (InterruptedException e) {
+ // Accepted work should finish before output is closed, but an interrupted waiter must
+ // not hold up the transform or the pipeline.
+ interrupted = true;
+ }
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void checkFailure() throws HopException {
+ synchronized (lock) {
+ if (failure != null) {
+ failureReported = true;
+ throw new HopException("Asynchronous processing failed", failure);
+ }
+ }
+ }
+
+ private void finish() throws HopException {
+ awaitJobsUncapped();
+ checkFailure();
setOutputDone();
}
+ @Override
+ public void dispose() {
+ // The common pool is shared. Drain our work, but never shut the pool down.
+ awaitJobs();
+ final boolean shouldLog;
+ synchronized (lock) {
+ // Only log a failure we have not already surfaced through the run loop, so it is not logged
+ // twice; the error accounting below still applies in either case.
+ shouldLog = failure != null && !failureReported;
+ if (shouldLog) {
+ failureReported = true;
+ }
+ }
+ if (failure != null) {
+ if (shouldLog) {
+ logError("Asynchronous processing failed", failure);
+ }
+ setErrors(1);
+ stopAll();
+ }
+ super.dispose();
+ }
+
@Override
public boolean processRow() throws HopException {
+ checkFailure();
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) { // no more input to be expected...
finish();
@@ -101,12 +191,8 @@ public boolean processRow() throws HopException {
sentenceBuilder = parsedSentenceBuilder().includePartOfSpeech(meta.isIncludePartOfSpeech());
} // End If first
- boolean sendToErrorRow = false;
- String errorMessage = null;
-
- process(r);
-
try {
+ process(r);
if (isRowLevel()) {
logRowlevel(
BaseMessages.getString(
@@ -115,29 +201,15 @@ public boolean processRow() throws HopException {
getLinesRead() + " : " + getInputRowMeta().getString(r)));
}
} catch (Exception e) {
- if (getTransformMeta().isDoingErrorHandling()) {
- sendToErrorRow = true;
- errorMessage = e.toString();
- } else {
- logError(
- BaseMessages.getString(PKG, "StanfordSimpleNlp.ErrorInTransformRunning")
- + e.getMessage());
- setErrors(1);
- stopAll();
- setOutputDone(); // signal end to receiver(s)
- return false;
- }
- if (sendToErrorRow) {
- // Simply add this row to the error row
- putError(
- getInputRowMeta(), r, 1, errorMessage, meta.getCorpusField(), "StanfordSimpleNlp001");
- }
+ // An infrastructure/worker failure must not be attributed to the current input row.
+ checkFailure();
+ handleRowError(r, e);
}
return true;
}
- private void process(Object[] inputRow) {
+ private void process(Object[] inputRow) throws HopException {
String corpus = trim((String) inputRow[data.indexOfCorpusField]);
if (isBlank(corpus)) {
return;
@@ -149,31 +221,63 @@ private void process(Object[] inputRow) {
}
}
- private void processAsync(String corpus, Object[] inputRow) {
+ private void handleRowError(Object[] row, Exception e) throws HopException {
+ if (!getTransformMeta().isDoingErrorHandling()) {
+ throw new HopException("Error processing row", e);
+ }
+ // Error output has the same serialization requirement as normal output.
+ synchronized (this) {
+ putError(
+ getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(), "StanfordSimpleNlp001");
+ }
+ }
+
+ private void processAsync(String text, Object[] inputRow) throws HopException {
synchronized (lock) {
- while (jobs.get() > maxJobs) {
+ while (jobs >= maxJobs && failure == null) {
try {
lock.wait();
} catch (InterruptedException e) {
- throw new HopRuntimeException(e);
+ Thread.currentThread().interrupt();
+ throw new HopException("Interrupted waiting for async processing", e);
}
}
+ checkFailure();
+ jobs++;
}
- executor.submit(
- () -> {
- try {
- processSync(corpus, inputRow);
- jobs.decrementAndGet();
- synchronized (lock) {
- lock.notifyAll();
+ try {
+ executor.submit(
+ () -> {
+ try {
+ try {
+ processSync(text, inputRow);
+ } catch (Exception e) {
+ handleRowError(inputRow, e);
+ }
+ } catch (Throwable e) {
+ synchronized (lock) {
+ if (failure == null) {
+ failure = e;
+ }
+ }
+ } finally {
+ synchronized (lock) {
+ jobs--;
+ lock.notifyAll();
+ }
}
- } catch (Exception e) {
- logError(e.getMessage(), e);
- }
- });
-
- jobs.incrementAndGet();
+ });
+ } catch (RuntimeException | Error e) {
+ synchronized (lock) {
+ jobs--;
+ if (failure == null) {
+ failure = e;
+ }
+ lock.notifyAll();
+ }
+ checkFailure();
+ }
}
private void processSync(String corpus, Object[] inputRow) {
diff --git a/plugins/transforms/stanfordnlp/src/test/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlpAsyncTest.java b/plugins/transforms/stanfordnlp/src/test/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlpAsyncTest.java
new file mode 100644
index 00000000000..d1d55236183
--- /dev/null
+++ b/plugins/transforms/stanfordnlp/src/test/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlpAsyncTest.java
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.stanford.nlp.simple;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.lang.reflect.Field;
+import java.util.List;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+@Timeout(10)
+class StanfordSimpleNlpAsyncTest {
+ private StanfordSimpleNlp transform;
+ private TransformMeta transformMeta;
+ private final Object[] row = {"text
"};
+
+ @BeforeEach
+ void setUp() throws Exception {
+ org.apache.hop.core.logging.HopLogStore.init();
+ StanfordSimpleNlpMeta meta = new StanfordSimpleNlpMeta();
+ meta.setCorpusField("input");
+ meta.setParallelism(true);
+ StanfordSimpleNlpData data = new StanfordSimpleNlpData();
+ data.outputRowMeta = new RowMeta();
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ transformMeta = spy(new TransformMeta("test", meta));
+ pipelineMeta.addTransform(transformMeta);
+ transform = spy(new StanfordSimpleNlp(transformMeta, meta, data, 0, pipelineMeta, null));
+ transform.first = false;
+ data.indexOfCorpusField = 0;
+ meta.setOutputFieldNamePrefix("");
+ for (int i = 0; i < data.sentenceFields.length; i++) {
+ data.indexOfMap.put(data.sentenceFields[i], i + 1);
+ }
+ ParsedSentence.Builder builder = mock(ParsedSentence.Builder.class);
+ ParsedSentence sentence = mock(ParsedSentence.class);
+ when(sentence.getInputRow()).thenReturn(row);
+ when(sentence.getSentenceText()).thenReturn("text");
+ when(builder.buildSentences(any(), anyString())).thenReturn(List.of(sentence));
+ setField("sentenceBuilder", builder);
+
+ doReturn(row, (Object[]) null).when(transform).getRow();
+ doNothing().when(transform).setOutputDone();
+ doNothing().when(transform).putRow(any(), any());
+ }
+
+ @Test
+ void inlineCompletionDoesNotLeavePhantomJob() throws Exception {
+ setField("executor", new InlineExecutor());
+ assertTrue(transform.processRow());
+ assertFalse(transform.processRow());
+ verify(transform).putRow(any(), any());
+ verify(transform).setOutputDone();
+ }
+
+ @Test
+ void workerExceptionIsVisibleAtEof() throws Exception {
+ setField("executor", new InlineExecutor());
+ doAnswer(
+ invocation -> {
+ throw new HopTransformException("worker failed");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ HopException failure = assertThrows(HopException.class, transform::processRow);
+ assertTrue(failure.getMessage().contains("Asynchronous processing failed"));
+ }
+
+ @Test
+ void workerErrorAlsoReleasesReservation() throws Exception {
+ setField("executor", new InlineExecutor());
+ doAnswer(
+ invocation -> {
+ throw new AssertionError("worker error");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertThrows(HopException.class, transform::processRow);
+ // Disposal must not hang even when submit captured an Error in its Future.
+ doNothing().when(transform).stopAll();
+ transform.dispose();
+ assertEquals(1, transform.getErrors());
+ }
+
+ @Test
+ void rejectedSubmissionRollsBackReservation() throws Exception {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ executor.shutdown();
+ setField("executor", executor);
+ assertThrows(HopException.class, transform::processRow);
+ doNothing().when(transform).stopAll();
+ transform.dispose();
+ assertEquals(1, transform.getErrors());
+ }
+
+ @Test
+ void synchronousFailureUsesConfiguredErrorHandling() throws Exception {
+ setParallelism(false);
+ doReturn(true).when(transformMeta).isDoingErrorHandling();
+ doAnswer(
+ invocation -> {
+ throw new HopTransformException("bad row");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ doNothing()
+ .when(transform)
+ .putError(
+ any(),
+ any(),
+ org.mockito.ArgumentMatchers.anyLong(),
+ anyString(),
+ anyString(),
+ anyString());
+ assertTrue(transform.processRow());
+ verify(transform)
+ .putError(
+ any(),
+ any(),
+ org.mockito.ArgumentMatchers.anyLong(),
+ anyString(),
+ anyString(),
+ anyString());
+ }
+
+ @Test
+ void asynchronousFailureUsesConfiguredErrorHandling() throws Exception {
+ setField("executor", new InlineExecutor());
+ doReturn(true).when(transformMeta).isDoingErrorHandling();
+ doAnswer(
+ invocation -> {
+ throw new HopTransformException("bad async row");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ doNothing()
+ .when(transform)
+ .putError(
+ any(),
+ any(),
+ org.mockito.ArgumentMatchers.anyLong(),
+ anyString(),
+ anyString(),
+ anyString());
+ assertTrue(transform.processRow());
+ assertFalse(transform.processRow());
+ verify(transform)
+ .putError(
+ any(),
+ any(),
+ org.mockito.ArgumentMatchers.anyLong(),
+ anyString(),
+ anyString(),
+ anyString());
+ }
+
+ @Test
+ void failureWakesAdmissionAtCapacity() throws Exception {
+ ExecutorService workers = Executors.newSingleThreadExecutor();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ setField("executor", workers);
+ setField("maxJobs", 1);
+ doReturn(row).when(transform).getRow();
+ doAnswer(
+ invocation -> {
+ entered.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ throw new HopTransformException("worker failed while admission waits");
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ CountDownLatch admitting = new CountDownLatch(1);
+ Future> nextRow =
+ caller.submit(
+ () -> {
+ admitting.countDown();
+ return assertThrows(HopException.class, transform::processRow);
+ });
+ assertTrue(admitting.await(5, TimeUnit.SECONDS));
+ assertFalse(nextRow.isDone());
+ release.countDown();
+ nextRow.get(5, TimeUnit.SECONDS);
+ doNothing().when(transform).stopAll();
+ transform.dispose();
+ assertEquals(1, transform.getErrors());
+ } finally {
+ release.countDown();
+ workers.shutdownNow();
+ caller.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ void eofDrainsAcceptedWorkBeforeOutputDone() throws Exception {
+ assertDrains(false);
+ }
+
+ @Test
+ void disposalDrainsAcceptedWorkWithoutShuttingDownExecutor() throws Exception {
+ assertDrains(true);
+ }
+
+ private void assertDrains(boolean dispose) throws Exception {
+ ExecutorService workers = Executors.newSingleThreadExecutor();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicInteger outputs = new AtomicInteger();
+ try {
+ setField("executor", workers);
+ doAnswer(
+ invocation -> {
+ entered.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ outputs.incrementAndGet();
+ return null;
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ CountDownLatch draining = new CountDownLatch(1);
+ Future> completion =
+ caller.submit(
+ () -> {
+ draining.countDown();
+ if (dispose) {
+ transform.dispose();
+ } else {
+ try {
+ assertFalse(transform.processRow());
+ } catch (HopException e) {
+ throw new AssertionError(e);
+ }
+ }
+ });
+ assertTrue(draining.await(5, TimeUnit.SECONDS));
+ assertFalse(completion.isDone());
+ release.countDown();
+ completion.get(5, TimeUnit.SECONDS);
+ assertEquals(1, outputs.get());
+ assertFalse(workers.isShutdown());
+ if (!dispose) {
+ verify(transform).setOutputDone();
+ }
+ } finally {
+ release.countDown();
+ workers.shutdownNow();
+ caller.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ void eofWaitsForSlowInFlightJobAndDeliversRowBeforeOutputDone() throws Exception {
+ // Regression: the 30 s drain cap used to apply on the normal end-of-input path too, so finish()
+ // abandoned jobs still running after 30 s and dropped their rows. finish() must wait without a
+ // cap and only then close the output.
+ ExecutorService workers = Executors.newSingleThreadExecutor();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicInteger outputs = new AtomicInteger();
+ try {
+ setField("executor", workers);
+ doAnswer(
+ invocation -> {
+ entered.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ outputs.incrementAndGet();
+ return null;
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ // Run the EOF step on its own thread so we can observe that, while a job is still in flight,
+ // it keeps waiting instead of dropping the job and closing the output on a short drain cap.
+ CountDownLatch draining = new CountDownLatch(1);
+ Future> completion =
+ caller.submit(
+ () -> {
+ draining.countDown();
+ try {
+ assertFalse(transform.processRow());
+ } catch (HopException e) {
+ throw new AssertionError(e);
+ }
+ });
+ assertTrue(draining.await(5, TimeUnit.SECONDS));
+ // The in-flight job is still running; the EOF path must not have returned (closed the output)
+ // yet, even though it has been waiting longer than any sub-second drain cap.
+ Thread.sleep(100);
+ assertFalse(completion.isDone());
+ release.countDown();
+ completion.get(5, TimeUnit.SECONDS);
+ assertEquals(1, outputs.get());
+ verify(transform).setOutputDone();
+ assertFalse(workers.isShutdown());
+ } finally {
+ release.countDown();
+ workers.shutdownNow();
+ caller.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ void eofHonoursStopToShortCircuitUncappedWait() throws Exception {
+ // Regression: finish() waits without a cap for in-flight jobs, so a stop must still break the
+ // wait and close the output, otherwise a never-completing job would hang the transform.
+ ExecutorService workers = Executors.newSingleThreadExecutor();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ setField("executor", workers);
+ doAnswer(
+ invocation -> {
+ entered.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ return null;
+ })
+ .when(transform)
+ .putRow(any(), any());
+ assertTrue(transform.processRow());
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ doReturn(true).when(transform).isStopped();
+ assertFalse(transform.processRow());
+ verify(transform).setOutputDone();
+ } finally {
+ release.countDown();
+ workers.shutdownNow();
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ private void setParallelism(boolean parallelism) {
+ ((StanfordSimpleNlpMeta) transformMeta.getTransform()).setParallelism(parallelism);
+ }
+
+ private void setField(String name, Object value) throws Exception {
+ Field field = StanfordSimpleNlp.class.getDeclaredField(name);
+ field.setAccessible(true);
+ field.set(transform, value);
+ }
+
+ private static final class InlineExecutor extends AbstractExecutorService {
+ @Override
+ public void execute(Runnable command) {
+ command.run();
+ }
+
+ @Override
+ public void shutdown() {}
+
+ @Override
+ public List shutdownNow() {
+ return List.of();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return false;
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return false;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) {
+ return false;
+ }
+ }
+}