diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java index 0dc52b19c9a..bf569b32e74 100644 --- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java +++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java @@ -31,9 +31,7 @@ import java.util.Objects; import java.util.SortedMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.exception.HopRuntimeException; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.i18n.BaseMessages; @@ -61,30 +59,121 @@ public DetectLanguage( private final int cores = Runtime.getRuntime().availableProcessors(); private final int maxJobs = cores * 100; private final ExecutorService executor = commonPool(); - private final AtomicInteger jobs = new AtomicInteger(0); + + /** How long draining waits for accepted asynchronous work before abandoning it. */ + private static final long DRAIN_TIMEOUT_MILLIS = 30_000L; + + // Guarded by lock, including reservation before submission. + private int jobs; + private Throwable failure; + // Guards against surfacing the same asynchronous failure twice (finish/checkFailure and dispose). + private boolean failureReported; private final Object lock = new Object(); private final LanguageDetector detector = fromAllSpokenLanguages().withPreloadedLanguageModels().build(); - private void finish() { - // Wait until all jobs are finished. + private void awaitJobs() { + awaitJobs(DRAIN_TIMEOUT_MILLIS); + } + + private void awaitJobs(final long timeoutMillis) { + boolean interrupted = false; + final long deadline = System.currentTimeMillis() + timeoutMillis; synchronized (lock) { - while (jobs.get() > 0) { + long remaining = timeoutMillis; + while (jobs > 0 && remaining > 0) { try { - lock.wait(); + lock.wait(remaining); } catch (InterruptedException e) { - throw new HopRuntimeException(e); + // Accepted work should finish before output is closed or resources are disposed, but an + // interrupted waiter must not hold up the transform or the pipeline. + interrupted = true; } + remaining = deadline - System.currentTimeMillis(); + } + if (jobs > 0) { + // A worker is stuck and no provider timeout applies. Do not block disposal forever: log and + // abandon the remaining jobs. + logBasic( + "Abandoning " + + jobs + + " asynchronous job(s) that did not finish within " + + timeoutMillis + + " ms"); } } - detector.unloadLanguageModels(); + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + /** + * Wait for every accepted job to complete on the normal end-of-input path. Unlike {@link + * #awaitJobs(long)} there is no timeout: rows that are merely slow must not be lost because the + * output is closed while workers still run. A stop (see {@link #stopRunning()}) breaks the wait. + */ + private void awaitJobsUncapped() { + boolean interrupted = false; + synchronized (lock) { + while (jobs > 0 && !isStopped()) { + try { + lock.wait(1_000L); + } catch (InterruptedException e) { + // Accepted work should finish before output is closed, but an interrupted waiter must + // not hold up the transform or the pipeline. + interrupted = true; + } + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private void checkFailure() throws HopException { + synchronized (lock) { + if (failure != null) { + failureReported = true; + throw new HopException("Asynchronous processing failed", failure); + } + } + } + + private void finish() throws HopException { + awaitJobsUncapped(); + checkFailure(); setOutputDone(); } + @Override + public void dispose() { + // The common pool is shared. Drain our work, but never shut the pool down. + awaitJobs(); + detector.unloadLanguageModels(); + final boolean shouldLog; + synchronized (lock) { + // Only log a failure we have not already surfaced through the run loop, so it is not logged + // twice; the error accounting below still applies in either case. + shouldLog = failure != null && !failureReported; + if (shouldLog) { + failureReported = true; + } + } + if (failure != null) { + if (shouldLog) { + logError("Asynchronous processing failed", failure); + } + setErrors(1); + stopAll(); + } + super.dispose(); + } + @Override public boolean processRow() throws HopException { + checkFailure(); Object[] r = getRow(); // Get row from input rowset & set row busy! if (r == null) { // no more input to be expected... finish(); @@ -108,12 +197,8 @@ public boolean processRow() throws HopException { cacheIndexPositions(); } // End If first - boolean sendToErrorRow = false; - String errorMessage = null; - - process(r); - try { + process(r); if (isRowLevel()) { logRowlevel( BaseMessages.getString( @@ -122,27 +207,15 @@ public boolean processRow() throws HopException { getLinesRead() + " : " + getInputRowMeta().getString(r))); } } catch (Exception e) { - if (getTransformMeta().isDoingErrorHandling()) { - sendToErrorRow = true; - errorMessage = e.toString(); - } else { - logError( - BaseMessages.getString(PKG, "DetectLanguage.ErrorInTransformRunning") + e.getMessage()); - setErrors(1); - stopAll(); - setOutputDone(); // signal end to receiver(s) - return false; - } - if (sendToErrorRow) { - // Simply add this row to the error row - putError(getInputRowMeta(), r, 1, errorMessage, meta.getCorpusField(), "DetectLanguage001"); - } + // An infrastructure/worker failure must not be attributed to the current input row. + checkFailure(); + handleRowError(r, e); } return true; } - private void process(Object[] inputRow) throws HopTransformException { + private void process(Object[] inputRow) throws HopException { String corpus = trim((String) inputRow[data.indexOfCorpusField]); if (isBlank(corpus)) { return; @@ -155,32 +228,62 @@ private void process(Object[] inputRow) throws HopTransformException { } } - private void processAsync(String corpus, Object[] inputRow) { + private void handleRowError(Object[] row, Exception e) throws HopException { + if (!getTransformMeta().isDoingErrorHandling()) { + throw new HopException("Error processing row", e); + } + // Error output has the same serialization requirement as normal output. + synchronized (this) { + putError(getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(), "DetectLanguage001"); + } + } + + private void processAsync(String text, Object[] inputRow) throws HopException { synchronized (lock) { - while (jobs.get() > maxJobs) { + while (jobs >= maxJobs && failure == null) { try { lock.wait(); } catch (InterruptedException e) { - throw new HopRuntimeException(e); + Thread.currentThread().interrupt(); + throw new HopException("Interrupted waiting for async processing", e); } } + checkFailure(); + jobs++; } - executor.submit( - () -> { - try { - processSync(corpus, inputRow); - jobs.decrementAndGet(); - - synchronized (lock) { - lock.notifyAll(); + try { + executor.submit( + () -> { + try { + try { + processSync(text, inputRow); + } catch (Exception e) { + handleRowError(inputRow, e); + } + } catch (Throwable e) { + synchronized (lock) { + if (failure == null) { + failure = e; + } + } + } finally { + synchronized (lock) { + jobs--; + lock.notifyAll(); + } } - } catch (Exception e) { - logError(e.getMessage(), e); - } - }); - - jobs.incrementAndGet(); + }); + } catch (RuntimeException | Error e) { + synchronized (lock) { + jobs--; + if (failure == null) { + failure = e; + } + lock.notifyAll(); + } + checkFailure(); + } } @SuppressWarnings( diff --git a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java new file mode 100644 index 00000000000..3806cae0642 --- /dev/null +++ b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.language; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.RETURNS_SELF; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.github.pemistahl.lingua.api.LanguageDetector; +import com.github.pemistahl.lingua.api.LanguageDetectorBuilder; +import java.lang.reflect.Field; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.MockedStatic; + +@Timeout(10) +class DetectLanguageAsyncTest { + private DetectLanguage transform; + private TransformMeta transformMeta; + private final Object[] row = {"

text

"}; + + @BeforeEach + void setUp() throws Exception { + org.apache.hop.core.logging.HopLogStore.init(); + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.setCorpusField("input"); + meta.setParallelism(true); + DetectLanguageData data = new DetectLanguageData(); + data.outputRowMeta = new RowMeta(); + PipelineMeta pipelineMeta = new PipelineMeta(); + transformMeta = spy(new TransformMeta("test", meta)); + pipelineMeta.addTransform(transformMeta); + try (MockedStatic factory = + mockStatic(LanguageDetectorBuilder.class)) { + LanguageDetectorBuilder builder = mock(LanguageDetectorBuilder.class, RETURNS_SELF); + LanguageDetector detector = mock(LanguageDetector.class); + factory.when(LanguageDetectorBuilder::fromAllSpokenLanguages).thenReturn(builder); + when(builder.build()).thenReturn(detector); + when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new TreeMap<>()); + transform = spy(new DetectLanguage(transformMeta, meta, data, 0, pipelineMeta, null)); + } + transform.first = false; + data.indexOfCorpusField = 0; + data.indexOfDetectedLanguage = 1; + data.indexOfDetectedLanguageConfidence = 2; + + doReturn(row, (Object[]) null).when(transform).getRow(); + doNothing().when(transform).setOutputDone(); + doNothing().when(transform).putRow(any(), any()); + } + + @Test + void inlineCompletionDoesNotLeavePhantomJob() throws Exception { + setField("executor", new InlineExecutor()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform).putRow(any(), any()); + verify(transform).setOutputDone(); + } + + @Test + void workerExceptionIsVisibleAtEof() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new HopTransformException("worker failed"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + HopException failure = assertThrows(HopException.class, transform::processRow); + assertTrue(failure.getMessage().contains("Asynchronous processing failed")); + } + + @Test + void workerErrorAlsoReleasesReservation() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new AssertionError("worker error"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertThrows(HopException.class, transform::processRow); + // Disposal must not hang even when submit captured an Error in its Future. + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void rejectedSubmissionRollsBackReservation() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + executor.shutdown(); + setField("executor", executor); + assertThrows(HopException.class, transform::processRow); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void synchronousFailureUsesConfiguredErrorHandling() throws Exception { + setParallelism(false); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void asynchronousFailureUsesConfiguredErrorHandling() throws Exception { + setField("executor", new InlineExecutor()); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad async row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void failureWakesAdmissionAtCapacity() throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + setField("executor", workers); + setField("maxJobs", 1); + doReturn(row).when(transform).getRow(); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + throw new HopTransformException("worker failed while admission waits"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch admitting = new CountDownLatch(1); + Future nextRow = + caller.submit( + () -> { + admitting.countDown(); + return assertThrows(HopException.class, transform::processRow); + }); + assertTrue(admitting.await(5, TimeUnit.SECONDS)); + assertFalse(nextRow.isDone()); + release.countDown(); + nextRow.get(5, TimeUnit.SECONDS); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void eofDrainsAcceptedWorkBeforeOutputDone() throws Exception { + assertDrains(false); + } + + @Test + void disposalDrainsAcceptedWorkWithoutShuttingDownExecutor() throws Exception { + assertDrains(true); + } + + private void assertDrains(boolean dispose) throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger outputs = new AtomicInteger(); + try { + setField("executor", workers); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + outputs.incrementAndGet(); + return null; + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch draining = new CountDownLatch(1); + Future completion = + caller.submit( + () -> { + draining.countDown(); + if (dispose) { + transform.dispose(); + } else { + try { + assertFalse(transform.processRow()); + } catch (HopException e) { + throw new AssertionError(e); + } + } + }); + assertTrue(draining.await(5, TimeUnit.SECONDS)); + assertFalse(completion.isDone()); + release.countDown(); + completion.get(5, TimeUnit.SECONDS); + assertEquals(1, outputs.get()); + assertFalse(workers.isShutdown()); + if (!dispose) { + verify(transform).setOutputDone(); + } + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void eofWaitsForSlowInFlightJobAndDeliversRowBeforeOutputDone() throws Exception { + // Regression: the 30 s drain cap used to apply on the normal end-of-input path too, so finish() + // abandoned jobs still running after 30 s and dropped their rows. finish() must wait without a + // cap and only then close the output. + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger outputs = new AtomicInteger(); + try { + setField("executor", workers); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + outputs.incrementAndGet(); + return null; + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + // Run the EOF step on its own thread so we can observe that, while a job is still in flight, + // it keeps waiting instead of dropping the job and closing the output on a short drain cap. + CountDownLatch draining = new CountDownLatch(1); + Future completion = + caller.submit( + () -> { + draining.countDown(); + try { + assertFalse(transform.processRow()); + } catch (HopException e) { + throw new AssertionError(e); + } + }); + assertTrue(draining.await(5, TimeUnit.SECONDS)); + // The in-flight job is still running; the EOF path must not have returned (closed the output) + // yet, even though it has been waiting longer than any sub-second drain cap. + Thread.sleep(100); + assertFalse(completion.isDone()); + release.countDown(); + completion.get(5, TimeUnit.SECONDS); + assertEquals(1, outputs.get()); + verify(transform).setOutputDone(); + assertFalse(workers.isShutdown()); + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void eofHonoursStopToShortCircuitUncappedWait() throws Exception { + // Regression: finish() waits without a cap for in-flight jobs, so a stop must still break the + // wait and close the output, otherwise a never-completing job would hang the transform. + ExecutorService workers = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + setField("executor", workers); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + return null; + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + doReturn(true).when(transform).isStopped(); + assertFalse(transform.processRow()); + verify(transform).setOutputDone(); + } finally { + release.countDown(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private void setParallelism(boolean parallelism) { + ((DetectLanguageMeta) transformMeta.getTransform()).setParallelism(parallelism); + } + + private void setField(String name, Object value) throws Exception { + Field field = DetectLanguage.class.getDeclaredField(name); + field.setAccessible(true); + field.set(transform, value); + } + + private static final class InlineExecutor extends AbstractExecutorService { + @Override + public void execute(Runnable command) { + command.run(); + } + + @Override + public void shutdown() {} + + @Override + public List shutdownNow() { + return List.of(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return false; + } + } +} diff --git a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java index a8590192563..a7325624c47 100644 --- a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java +++ b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java @@ -18,16 +18,23 @@ package org.apache.hop.pipeline.transforms.execprocess; import java.io.BufferedReader; +import java.io.Closeable; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.CountDownLatch; +import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.StringUtils; import org.apache.hop.core.Const; import org.apache.hop.core.exception.HopException; @@ -44,8 +51,13 @@ public class ExecProcess extends BaseTransform private static final Class PKG = ExecProcessMeta.class; public static final String CONST_EXEC_PROCESS_EXCEPTION_COULD_NOT_FIND_FIELD = "ExecProcess.Exception.CouldNotFindField"; - private boolean killing; - private CountDownLatch waitForLatch; + + /** + * How long a stopped process is given to exit on a graceful destroy before it is force-killed. + */ + private static final long GRACE_PERIOD_SECONDS = 30; + + private final AtomicReference activeExecution = new AtomicReference<>(); public ExecProcess( TransformMeta transformMeta, @@ -95,13 +107,19 @@ public boolean processRow() throws HopException { cmdArray.add(Const.NVL(argString, "")); } - execProcess(cmdArray.toArray(new String[0]), processResult); + if (!execProcess(cmdArray.toArray(new String[0]), processResult)) { + setOutputDone(); + return false; + } } else { if (isDetailed() && !hasBalancedQuotes(processString)) { logDetailed( BaseMessages.getString(PKG, "ExecProcess.Log.UnbalancedQuotes", processString)); } - execProcess(tokenizeCommandLine(processString), processResult); + if (!execProcess(tokenizeCommandLine(processString), processResult)) { + setOutputDone(); + return false; + } } if (meta.isFailWhenNotSuccess() && processResult.getExitValue() != 0) { @@ -199,126 +217,198 @@ private void initializeAndLookup() throws HopException { @Override public void stopRunning() throws HopException { - if (waitForLatch != null) { - killing = true; - try { - waitForLatch.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new HopException("Interrupted exception while kill the process", e); - } + // A pipeline stop must not wait for a child process or a stream reader. In particular, a + // descendant can keep a pipe open after the direct child exits. + setStopped(true); + final var execution = activeExecution.get(); + if (execution != null) { + execution.cancelled.set(true); } } - private void execProcess(String[] process, ProcessResult processresult) throws HopException { - Process p = null; - waitForLatch = new CountDownLatch(1); + private boolean execProcess(final String[] command, final ProcessResult result) + throws HopException { + final var execution = new ProcessExecution(); + activeExecution.set(execution); + Process child = null; + boolean completed = false; try { - String errorMsg = null; - // execute process + if (isStopped()) { + return false; + } try { - p = data.runtime.exec(process); + child = data.runtime.exec(command); } catch (Exception e) { - errorMsg = e.getMessage(); + // An exec failure may be an IOException, SecurityException or IllegalArgumentException; all + // must reach the error hop rather than escape processRow as a non-HopException. + if (isStopped() || execution.cancelled.get()) { + return false; + } + result.setErrorStream(e.getMessage()); + return true; } - if (p == null) { - processresult.setErrorStream(errorMsg); - } else { - final Process child = p; - // Drain stdout/stderr while the process runs. Reading only after onExit() can deadlock: - // if the child fills either pipe buffer before exiting, it blocks on write forever. - CompletableFuture stdoutFuture = - CompletableFuture.supplyAsync( - () -> { - try { - return getOutputString( - new BufferedReader(new InputStreamReader(child.getInputStream()))); - } catch (IOException e) { - throw new CompletionException(e); - } - }); - CompletableFuture stderrFuture = - CompletableFuture.supplyAsync( - () -> { - try { - return getOutputString( - new BufferedReader(new InputStreamReader(child.getErrorStream()))); - } catch (IOException e) { - throw new CompletionException(e); - } - }); - - while (true) { - try { - if (child.waitFor(1, TimeUnit.SECONDS)) { - break; - } - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new HopException( - "Interrupted exception while running the process " + Arrays.toString(process) + "!", - ie); - } - if (killing) { - child.children().forEach(ProcessHandle::destroy); - if (child.isAlive()) { - child.destroy(); - } - try { - if (!child.waitFor(30, TimeUnit.SECONDS)) { - child.destroyForcibly(); - child.waitFor(); - } - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new HopException( - "Interrupted exception while running the process " - + Arrays.toString(process) - + "!", - ie); - } - logMinimal(BaseMessages.getString(PKG, "ExecProcess.AbortProcess")); - break; - } + if (isCancelled(execution)) { + return false; + } + final Process process = child; + // Owned daemon readers keep blocked pipes out of the shared common pool. They close their + // streams on normal completion; cancellation also arranges asynchronous stream cleanup. + final var stdout = execution.readers.submit(() -> readOutput(process.getInputStream())); + final var stderr = execution.readers.submit(() -> readOutput(process.getErrorStream())); + while (true) { + // Remember children while the parent is alive: after it exits, inherited pipes may + // still be held by descendants that are no longer discoverable through that parent. + try (final var descendants = process.descendants()) { + descendants.forEach(execution.descendants::add); } - - try { - processresult.setOutputStream(stdoutFuture.get()); - processresult.setErrorStream(stderrFuture.get()); - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof IOException ioe) { - throw ioe; - } - if (cause instanceof RuntimeException re) { - throw re; - } - throw new IOException(cause); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new HopException( - "Interrupted exception while running the process " + Arrays.toString(process) + "!", - ie); + if (isCancelled(execution)) { + return false; } - - processresult.setExitValue(child.exitValue()); + // Poll at a second's granularity so we do not rescan /proc more often than needed. + if (process.waitFor(1, TimeUnit.SECONDS)) { + break; + } + } + final var output = awaitOutput(stdout, execution); + if (isCancelled(execution)) { + return false; + } + final var error = awaitOutput(stderr, execution); + if (isCancelled(execution)) { + return false; } - } catch (IOException ioe) { - throw new HopException( - "IO exception while running the process " + Arrays.toString(process) + "!", ioe); - } catch (Exception e) { - throw new HopException(e); + result.setOutputStream(output); + result.setErrorStream(error); + result.setExitValue(process.exitValue()); + completed = true; + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new HopException("Interrupted while running process " + Arrays.toString(command), e); + } catch (ExecutionException e) { + throw new HopException("Unable to read process output", e.getCause()); } finally { - if (killing || waitForLatch.getCount() > 0) { - waitForLatch.countDown(); - killing = false; + activeExecution.compareAndSet(execution, null); + execution.readers.shutdownNow(); + if (child != null) { + if (completed) { + // Normal completion: the owned readers already closed the streams. Closing them again is + // a defensive no-op that needs no dedicated thread. + closeProcessStream(child.getOutputStream()); + closeProcessStream(child.getInputStream()); + closeProcessStream(child.getErrorStream()); + } else { + // Aborted: destroy the process (graceful then forced) inline on the transform thread so a + // hop-run shutdown that exits the JVM cannot orphan a still-running command, then close + // its streams on a background thread. + cleanupProcess(child, execution); + } } - if (p != null) { - p.destroy(); + } + } + + private boolean isCancelled(final ProcessExecution execution) { + return isStopped() || execution.cancelled.get(); + } + + private String awaitOutput(final Future output, final ProcessExecution execution) + throws InterruptedException, ExecutionException { + while (!isCancelled(execution)) { + try { + return output.get(100, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + // Observe stop requests even after the process exits but a stream drainer is still active. + } + } + return null; + } + + private String readOutput(final InputStream stream) throws IOException { + try (final var reader = new BufferedReader(new InputStreamReader(stream))) { + return getOutputString(reader); + } + } + + private void cleanupProcess(final Process process, final ProcessExecution execution) { + // Aborted: stop the process right away on the transform thread so a hop-run shutdown that calls + // System.exit() cannot orphan a still-running command. Only the stream close, that can block in + // native I/O or on a reader's monitor, is deferred to a background thread. + killProcess(process, execution); + closeStreamsAsync(process); + } + + /** + * Stop the process gracefully (SIGTERM) and, after a grace period, force it (SIGKILL). This runs + * on the transform thread before processRow returns, so a pipeline shutdown that exits the JVM + * cannot orphan a still-running command. The daemon cleanup that used to own this work could be + * killed by System.exit() before the forced kill ever ran. + */ + private void killProcess(final Process process, final ProcessExecution execution) { + try { + try (final var descendants = process.descendants()) { + descendants.forEach(execution.descendants::add); } + // Ask gracefully first (SIGTERM) so a script trapping it can clean up and roll back, then + // fall back to a forced kill (SIGKILL) only if it does not exit. + execution.descendants.forEach(ProcessHandle::destroy); + if (process.isAlive()) { + process.destroy(); + } + if (!process.waitFor(GRACE_PERIOD_SECONDS, TimeUnit.SECONDS)) { + execution.descendants.forEach(ProcessHandle::destroyForcibly); + if (process.isAlive()) { + process.destroyForcibly(); + } + } + } catch (InterruptedException e) { + // A forced kill still makes sure the process goes away without waiting again. + Thread.currentThread().interrupt(); + execution.descendants.forEach(ProcessHandle::destroyForcibly); + if (process.isAlive()) { + process.destroyForcibly(); + } + } + } + + /** + * Close the process streams on a daemon thread. stream.close() may block in native I/O or on a + * reader's monitor, so it must not run inline on the transform or pipeline thread. + */ + private void closeStreamsAsync(final Process process) { + final var closer = + new Thread( + () -> { + closeProcessStream(process.getOutputStream()); + closeProcessStream(process.getInputStream()); + closeProcessStream(process.getErrorStream()); + }, + "hop-execprocess-cleanup"); + closer.setDaemon(true); + closer.start(); + } + + private void closeProcessStream(final Closeable stream) { + try { + stream.close(); + } catch (IOException e) { + logDebug("Unable to close process stream", e); } } + private static final class ProcessExecution { + private final AtomicBoolean cancelled = new AtomicBoolean(); + // Owned by the executing (transform) thread, which also performs the graceful-then-forced kill. + private final Set descendants = new HashSet<>(); + private final ExecutorService readers = + Executors.newFixedThreadPool( + 2, + task -> { + final var thread = new Thread(task, "hop-execprocess-output"); + thread.setDaemon(true); + return thread; + }); + } + /** * Split a command line into the executable and its arguments. Tokens are separated by whitespace, * but a section wrapped in single or double quotes is kept together as one token even when it diff --git a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java index 9aae436f365..bea7009d75e 100644 --- a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java +++ b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java @@ -23,10 +23,21 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.util.Arrays; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Stream; import org.apache.hop.core.HopEnvironment; import org.apache.hop.core.IRowSet; import org.apache.hop.core.QueueRowSet; @@ -41,6 +52,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.condition.EnabledOnOs; import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.RegisterExtension; @@ -359,6 +371,250 @@ void init_requiresResultFieldName() throws HopException { assertFalse(transform.init()); } + @Test + @Timeout(10) + void stopWhileDrainingOutputDoesNotWaitForEndOfStream() throws Exception { + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + final var process = mock(Process.class); + final var stdout = new BlockingInputStream(); + when(process.getInputStream()).thenReturn(stdout); + when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream()); + when(process.waitFor(1, TimeUnit.SECONDS)).thenReturn(true); + when(process.descendants()).thenAnswer(invocation -> Stream.empty()); + when(runtime.exec(any(String[].class))).thenReturn(process); + transform.getData().runtime = runtime; + final var executor = Executors.newSingleThreadExecutor(); + try { + final var running = executor.submit(transform::processRow); + assertTrue(stdout.reading.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + transform.stopRunning(); + assertFalse(running.get(5, TimeUnit.SECONDS)); + assertTrue(stdout.closed.await(5, TimeUnit.SECONDS)); + assertEquals(0, transform.getErrors()); + assertEquals(0, transform.getOutputRowSets().getFirst().size()); + } finally { + stdout.close(); + executor.shutdownNow(); + } + } + + @Test + @Timeout(10) + void stopDuringLaunchCancelsTheNewProcessWithoutBlockingStopper() throws Exception { + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + final var process = mock(Process.class); + final var launching = new CountDownLatch(1); + final var releaseLaunch = new CountDownLatch(1); + final var destroyed = new CountDownLatch(1); + when(runtime.exec(any(String[].class))) + .thenAnswer( + invocation -> { + launching.countDown(); + assertTrue(releaseLaunch.await(5, TimeUnit.SECONDS)); + return process; + }); + when(process.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream()); + when(process.isAlive()).thenReturn(true); + when(process.descendants()).thenAnswer(invocation -> Stream.empty()); + when(process.destroyForcibly()) + .thenAnswer( + invocation -> { + destroyed.countDown(); + return process; + }); + transform.getData().runtime = runtime; + final var executor = Executors.newSingleThreadExecutor(); + try { + final var running = executor.submit(transform::processRow); + assertTrue(launching.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + releaseLaunch.countDown(); + assertFalse(running.get(5, TimeUnit.SECONDS)); + assertTrue(destroyed.await(5, TimeUnit.SECONDS)); + assertEquals(0, transform.getErrors()); + } finally { + releaseLaunch.countDown(); + executor.shutdownNow(); + } + } + + @Test + @Timeout(10) + void cancellationRetainsChildrenObservedBeforeParentExit() throws Exception { + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + final var process = mock(Process.class); + final var descendant = mock(ProcessHandle.class); + final var destroyed = new CountDownLatch(1); + final var stdout = new BlockingInputStream(); + when(descendant.destroyForcibly()) + .thenAnswer( + invocation -> { + destroyed.countDown(); + return true; + }); + when(process.descendants()).thenReturn(Stream.of(descendant), Stream.empty()); + when(process.getInputStream()).thenReturn(stdout); + when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream()); + when(process.waitFor(1, TimeUnit.SECONDS)).thenReturn(true); + when(runtime.exec(any(String[].class))).thenReturn(process); + transform.getData().runtime = runtime; + final var executor = Executors.newSingleThreadExecutor(); + try { + final var running = executor.submit(transform::processRow); + assertTrue(stdout.reading.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + assertFalse(running.get(5, TimeUnit.SECONDS)); + assertTrue(destroyed.await(5, TimeUnit.SECONDS)); + } finally { + stdout.close(); + executor.shutdownNow(); + } + } + + @Test + void stopBeforeLaunchDoesNotStartAProcess() throws Exception { + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + transform.getData().runtime = runtime; + transform.stopRunning(); + assertFalse(transform.processRow()); + verify(runtime, never()).exec(any(String[].class)); + } + + @Test + @Timeout(10) + void stoppedProcessThatExitsWithinGracePeriodIsDestroyedNotForceKilled() throws Exception { + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + final var process = mock(Process.class); + final var stdout = new BlockingInputStream(); + when(process.getInputStream()).thenReturn(stdout); + when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream()); + when(process.isAlive()).thenReturn(true); + when(process.descendants()).thenAnswer(invocation -> Stream.empty()); + // The process has not exited on its own while it is polled ... + when(process.waitFor(1, TimeUnit.SECONDS)).thenReturn(false); + // ... but a graceful destroy makes it exit within the grace period. + when(process.waitFor(30, TimeUnit.SECONDS)).thenReturn(true); + when(runtime.exec(any(String[].class))).thenReturn(process); + transform.getData().runtime = runtime; + final var executor = Executors.newSingleThreadExecutor(); + try { + final var running = executor.submit(transform::processRow); + assertTrue(stdout.reading.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + assertFalse(running.get(5, TimeUnit.SECONDS)); + // A graceful destroy must be tried first, and a forced kill never becomes necessary when the + // process cooperates within the grace period. + assertTrue(stdout.closed.await(5, TimeUnit.SECONDS)); + verify(process).destroy(); + verify(process, never()).destroyForcibly(); + } finally { + stdout.close(); + executor.shutdownNow(); + } + } + + @Test + @Timeout(10) + void forceKillRunsInlineOnTransformThreadBeforeProcessRowReturns() throws Exception { + // Regression: the forced kill used to run on a daemon cleanup thread that hop-run's + // System.exit() could kill before destroyForcibly() ever ran, leaving a SIGTERM-ignoring + // command running. The graceful-then-forced kill must now run inline on the transform thread so + // processRow() does not return until the process has been forcibly killed. + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + final var process = mock(Process.class); + final var stdout = new BlockingInputStream(); + final var forceKillEntered = new CountDownLatch(1); + final var releaseForceKill = new CountDownLatch(1); + when(process.getInputStream()).thenReturn(stdout); + when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream()); + when(process.isAlive()).thenReturn(true); + when(process.descendants()).thenAnswer(invocation -> Stream.empty()); + // The process never exits on its own, neither in the poll loop nor within the grace period. + when(process.waitFor(1, TimeUnit.SECONDS)).thenReturn(false); + when(process.waitFor(30, TimeUnit.SECONDS)).thenReturn(false); + when(process.destroyForcibly()) + .thenAnswer( + invocation -> { + forceKillEntered.countDown(); + assertTrue(releaseForceKill.await(5, TimeUnit.SECONDS)); + return process; + }); + when(runtime.exec(any(String[].class))).thenReturn(process); + transform.getData().runtime = runtime; + final var executor = Executors.newSingleThreadExecutor(); + try { + final var running = executor.submit(transform::processRow); + assertTrue(stdout.reading.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + assertTrue(forceKillEntered.await(5, TimeUnit.SECONDS)); + // processRow is still running because the forced kill holds the transform thread; it must not + // have returned and left the kill to a background thread that System.exit() could kill. + assertThrows(TimeoutException.class, () -> running.get(2, TimeUnit.SECONDS)); + releaseForceKill.countDown(); + assertFalse(running.get(5, TimeUnit.SECONDS)); + verify(process).destroy(); + verify(process).destroyForcibly(); + assertEquals(0, transform.getErrors()); + } finally { + releaseForceKill.countDown(); + stdout.close(); + executor.shutdownNow(); + } + } + + private static final class BlockingInputStream extends InputStream { + private final CountDownLatch reading = new CountDownLatch(1); + private final CountDownLatch closed = new CountDownLatch(1); + + @Override + public int read() { + reading.countDown(); + boolean interrupted = false; + while (true) { + try { + closed.await(); + break; + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + return -1; + } + + @Override + public void close() { + closed.countDown(); + } + } + private ExecProcess newTransform(ExecProcessMeta meta, IRowMeta rowMeta, Object[] row) { TransformMeta tm = smh.transformMeta; when(tm.isDoingErrorHandling()).thenReturn(false); diff --git a/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java b/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java index 0adaa88d9b5..82736e3ebd7 100644 --- a/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java +++ b/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java @@ -17,13 +17,11 @@ package org.apache.hop.pipeline.transforms.languagemodelchat; -import static java.lang.Runtime.getRuntime; import static java.lang.System.arraycopy; import static java.time.Duration.between; import static java.time.Instant.now; import static java.util.concurrent.Executors.newWorkStealingPool; import static java.util.concurrent.ForkJoinPool.commonPool; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.trim; @@ -37,7 +35,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopRuntimeException; @@ -62,11 +60,15 @@ public class LanguageModelChat extends BaseTransform facadeMap = new ConcurrentHashMap<>(); private LanguageModelChatMeta modelMeta; - private final int parallelism = 1; - private ForkJoinPool executor; + private volatile ExecutorService executor; + private volatile boolean ownsExecutor; private final int cores = Runtime.getRuntime().availableProcessors(); private final AtomicInteger jobs = new AtomicInteger(0); private final int maxJobs = cores * 100; + + /** How long draining waits for accepted asynchronous work before abandoning it. */ + private static final long DRAIN_TIMEOUT_MILLIS = 30_000L; + private final Object lock = new Object(); public LanguageModelChat( @@ -80,29 +82,68 @@ public LanguageModelChat( } private void finish() { + awaitJobsUncapped(); + shutdownOwnedExecutor(); + + setOutputDone(); + } + + private void awaitJobs() { + awaitJobs(DRAIN_TIMEOUT_MILLIS); + } + private void awaitJobs(long timeoutMillis) { + boolean interrupted = false; + final long deadline = System.currentTimeMillis() + timeoutMillis; synchronized (lock) { - while (jobs.get() > 0) { + long remaining = timeoutMillis; + while (jobs.get() > 0 && remaining > 0) { try { - lock.wait(); + lock.wait(remaining); } catch (InterruptedException e) { - throw new HopRuntimeException(e); + // Accepted work should finish before output is closed or resources are disposed, but an + // interrupted waiter must not hold up the transform or the pipeline forever. + interrupted = true; } + remaining = deadline - System.currentTimeMillis(); } + if (jobs.get() > 0) { + // A worker is stuck and no provider timeout applies. Do not block disposal forever: log and + // abandon the remaining jobs. + logBasic( + "Abandoning " + + jobs.get() + + " asynchronous job(s) that did not finish within " + + timeoutMillis + + " ms"); + } + } + if (interrupted) { + Thread.currentThread().interrupt(); } + } - if (executor != null) { - try { - while (executor.hasQueuedSubmissions() || executor.getActiveThreadCount() > 0) { - SECONDS.sleep(5); + /** + * Wait for every accepted job to complete on the normal end-of-input path. Unlike {@link + * #awaitJobs(long)} there is no timeout: rows that are merely slow must not be lost because the + * output is closed (and the facades dropped) while workers still run. A stop breaks the wait. + */ + private void awaitJobsUncapped() { + boolean interrupted = false; + synchronized (lock) { + while (jobs.get() > 0 && !isStopped()) { + try { + lock.wait(1_000L); + } catch (InterruptedException e) { + // Accepted work should finish before output is closed, but an interrupted waiter must + // not hold up the transform or the pipeline. + interrupted = true; } - } catch (InterruptedException e) { - logError(e.getMessage(), e); } } - - facadeMap.clear(); - setOutputDone(); + if (interrupted) { + Thread.currentThread().interrupt(); + } } @Override @@ -127,14 +168,11 @@ public boolean processRow() throws HopException { new LanguageModel(modelMeta), new LanguageModelFacade(variables, modelMeta, getLogChannel())); - int parallelism = - meta.getParallelism() <= 0 ? getRuntime().availableProcessors() : meta.getParallelism(); - if (parallelism != 0 && this.parallelism != parallelism || executor == null) { - if (executor != null) { - executor.shutdownNow(); - } - executor = (ForkJoinPool) newWorkStealingPool(parallelism); - } + ownsExecutor = meta.getParallelism() != 0; + executor = + ownsExecutor + ? newWorkStealingPool(meta.getParallelism() < 0 ? cores : meta.getParallelism()) + : commonPool(); if (isEmpty(meta.getInputField())) { throw new HopException( @@ -179,53 +217,93 @@ private void process(Object[] inputRow) { } synchronized (lock) { - while (jobs.get() > maxJobs) { + while (jobs.get() >= maxJobs && !isStopped()) { try { lock.wait(); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new HopRuntimeException(e); } } } - ExecutorService exec = meta.getParallelism() == 0 ? commonPool() : executor; - - exec.submit( - () -> { - try { - processSync(inputRow, message); - } catch (Exception e) { + if (isStopped()) { + return; + } + jobs.incrementAndGet(); + try { + executor.submit( + () -> { try { - if (getTransformMeta().isDoingErrorHandling()) { - String errorMessage = e.toString(); - putError( - getInputRowMeta(), - inputRow, - 1L, - errorMessage, - meta.getInputField(), - "LanguageModelChat001"); - } else { - logError(i18n("LanguageModelChat.ErrorInTransformRunning") + e.getMessage(), e); - setErrors(1L); - stopAll(); + processSync(inputRow, message); + } catch (Exception e) { + try { + if (getTransformMeta().isDoingErrorHandling()) { + String errorMessage = e.toString(); + putError( + getInputRowMeta(), + inputRow, + 1L, + errorMessage, + meta.getInputField(), + "LanguageModelChat001"); + } else { + logError(i18n("LanguageModelChat.ErrorInTransformRunning") + e.getMessage(), e); + setErrors(1L); + stopAll(); + } + } catch (HopTransformException ex) { + logError(e.getMessage(), e); + logError(ex.getMessage(), ex); + if (!getTransformMeta().isDoingErrorHandling()) { + setErrors(1L); + stopAll(); + } } - } catch (HopTransformException ex) { - logError(e.getMessage(), e); - logError(ex.getMessage(), ex); - if (!getTransformMeta().isDoingErrorHandling()) { - setErrors(1L); - stopAll(); - } - } - } finally { - jobs.decrementAndGet(); - synchronized (lock) { - lock.notifyAll(); + } finally { + jobFinished(); } - } - }); - jobs.incrementAndGet(); + }); + } catch (RejectedExecutionException e) { + jobFinished(); + if (!isStopped()) { + throw e; + } + } + } + + private void jobFinished() { + jobs.decrementAndGet(); + synchronized (lock) { + lock.notifyAll(); + } + } + + private void shutdownOwnedExecutor() { + if (ownsExecutor && executor != null) { + // Accepted tasks still drain. Never await this executor from a worker stopping the pipeline. + executor.shutdown(); + } + } + + @Override + public void stopRunning() throws HopException { + setStopped(true); + shutdownOwnedExecutor(); + synchronized (lock) { + lock.notifyAll(); + } + super.stopRunning(); + } + + @Override + public void dispose() { + shutdownOwnedExecutor(); + awaitJobs(); + // The runner disposes after processRow returns, which need not have reached normal EOF after + // a stop or failure. Keep facades alive until every accepted worker has left processSync. + facadeMap.clear(); + super.dispose(); } private void processSync(Object[] inputRow, String message) diff --git a/plugins/transforms/languagemodelchat/src/test/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChatLifecycleTest.java b/plugins/transforms/languagemodelchat/src/test/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChatLifecycleTest.java new file mode 100644 index 00000000000..6db2c2f69ad --- /dev/null +++ b/plugins/transforms/languagemodelchat/src/test/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChatLifecycleTest.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.languagemodelchat; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.model.chat.response.ChatResponse; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinWorkerThread; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.QueueRowSet; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.logging.ILoggingObject; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.pipeline.transform.RowAdapter; +import org.apache.hop.pipeline.transforms.languagemodelchat.internals.LanguageModelFacade; +import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.extension.RegisterExtension; + +@Timeout(15) +class LanguageModelChatLifecycleTest { + @RegisterExtension + static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); + + private TransformMockHelper helper; + + @BeforeEach + void setUp() throws Exception { + helper = + new TransformMockHelper<>("chat", LanguageModelChatMeta.class, LanguageModelChatData.class); + when(helper.logChannelFactory.create(any(), any(ILoggingObject.class))) + .thenReturn(helper.iLogChannel); + when(helper.pipeline.isRunning()).thenReturn(true); + HopEnvironment.init(); + } + + @AfterEach + void cleanUp() { + helper.cleanUp(); + } + + private LanguageModelChat transform(final int parallelism) { + final var meta = new LanguageModelChatMeta(); + meta.setDefault(); + meta.setMock(true); + meta.setMockOutputValue("answer"); + meta.setParallelism(parallelism); + final var transform = + new LanguageModelChat( + helper.transformMeta, + meta, + new LanguageModelChatData(), + 0, + helper.pipelineMeta, + helper.pipeline); + final var rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaString("input")); + transform.setInputRowMeta(rowMeta); + final var input = helper.getMockInputRowSet(new Object[] {"question"}); + when(input.getRowMeta()).thenReturn(rowMeta); + transform.addRowSetToInputRowSets(input); + transform.addRowSetToOutputRowSets(new QueueRowSet()); + return transform; + } + + @Test + void ownedPoolDrainsMockOutputBeforeCompletion() throws Exception { + final var transform = transform(2); + final var pool = new AtomicReference(); + transform.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(final IRowMeta rowMeta, final Object[] row) { + pool.set(((ForkJoinWorkerThread) Thread.currentThread()).getPool()); + } + }); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + final var output = transform.getOutputRowSets().getFirst(); + assertEquals(1, output.size()); + final var row = output.getRow(); + assertEquals("MOCK", row[transform.getData().indexOfFinishReason]); + assertEquals("answer", row[transform.getData().indexOfOutput]); + assertTrue(output.isDone()); + transform.dispose(); + transform.dispose(); + assertTrue(pool.get().isShutdown()); + assertTrue(pool.get().awaitTermination(5, TimeUnit.SECONDS)); + assertEquals(0, transform.getErrors()); + } + + @Test + void providerExceptionStillProducesAnErrorResultRow() throws Exception { + try (final var facades = + mockConstruction( + LanguageModelFacade.class, + (facade, context) -> { + when(facade.inputToChatMessages(any())).thenReturn(List.of()); + when(facade.chat(any())).thenThrow(new IllegalStateException("provider unavailable")); + })) { + final var transform = transform(1); + transform.getMeta().setMock(false); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + final var row = transform.getOutputRowSets().getFirst().getRow(); + assertEquals("ERROR", row[transform.getData().indexOfFinishReason]); + assertEquals("provider unavailable", row[transform.getData().indexOfOutput]); + assertEquals(0, transform.getErrors()); + transform.dispose(); + } + } + + @Test + void rejectionDuringStopIsNotReportedAsARowFailure() throws Exception { + final var executor = mock(ExecutorService.class); + try (final var factory = mockStatic(Executors.class)) { + factory.when(() -> Executors.newWorkStealingPool(1)).thenReturn(executor); + final var transform = transform(1); + when(executor.submit(any(Runnable.class))) + .thenAnswer( + invocation -> { + transform.stopRunning(); + throw new RejectedExecutionException("stopping"); + }); + assertTrue(transform.processRow()); + transform.dispose(); + assertEquals(0, transform.getErrors()); + } + } + + @Test + void commonPoolRemainsUsableAfterCompletionAndDisposal() throws Exception { + final var transform = transform(0); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + transform.dispose(); + transform.stopRunning(); + assertFalse(ForkJoinPool.commonPool().isShutdown()); + assertEquals(42, ForkJoinPool.commonPool().submit(() -> 42).get(5, TimeUnit.SECONDS)); + } + + @Test + void workerFailureCanStopThePipelineWithoutWaitingForItself() throws Exception { + final var transform = transform(1); + final var stopping = new CountDownLatch(1); + doAnswer( + invocation -> { + transform.stopRunning(); + stopping.countDown(); + return null; + }) + .when(helper.pipeline) + .stopAll(); + transform.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(final IRowMeta rowMeta, final Object[] row) + throws HopTransformException { + throw new HopTransformException("output failed"); + } + }); + assertTrue(transform.processRow()); + assertTrue(stopping.await(5, TimeUnit.SECONDS)); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void disposeWaitsForAcceptedOutputWithoutHoldingItsMonitor() throws Exception { + final var transform = transform(1); + final var outputStarted = new CountDownLatch(1); + final var releaseOutput = new CountDownLatch(1); + final var workerFinished = new AtomicBoolean(); + transform.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(final IRowMeta rowMeta, final Object[] row) + throws HopTransformException { + outputStarted.countDown(); + try { + if (!releaseOutput.await(5, TimeUnit.SECONDS)) { + throw new HopTransformException("output wait timed out"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new HopTransformException(e); + } + workerFinished.set(true); + } + }); + final var disposer = Executors.newSingleThreadExecutor(); + try { + assertTrue(transform.processRow()); + assertTrue(outputStarted.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + final var disposed = + disposer.submit( + () -> { + transform.dispose(); + assertTrue( + workerFinished.get(), "dispose returned before accepted output completed"); + }); + assertFalse(disposed.isDone()); + releaseOutput.countDown(); + disposed.get(5, TimeUnit.SECONDS); + assertEquals(0, transform.getErrors()); + } finally { + releaseOutput.countDown(); + disposer.shutdownNow(); + } + } + + @Test + void queuedWorkAfterStopRetainsFacadeAndDrainsOnDispose() throws Exception { + final var executor = Executors.newSingleThreadExecutor(); + final var queueBlocked = new CountDownLatch(1); + final var releaseQueue = new CountDownLatch(1); + final var workCompleted = new AtomicBoolean(); + final var blocker = + executor.submit( + () -> { + queueBlocked.countDown(); + assertTrue(releaseQueue.await(5, TimeUnit.SECONDS)); + return null; + }); + try (final var facades = + mockConstruction( + LanguageModelFacade.class, + (facade, context) -> + when(facade.inputToChatMessages(any())) + .thenAnswer( + invocation -> { + workCompleted.set(true); + return List.of(); + })); + final var factory = mockStatic(Executors.class, CALLS_REAL_METHODS)) { + assertTrue(queueBlocked.await(5, TimeUnit.SECONDS)); + factory.when(() -> Executors.newWorkStealingPool(1)).thenReturn(executor); + final var transform = transform(1); + assertTrue(transform.processRow()); + transform.stopRunning(); + assertFalse(transform.processRow()); + assertFalse(workCompleted.get()); + + releaseQueue.countDown(); + transform.dispose(); + assertTrue(workCompleted.get(), "dispose must drain accepted queued work after stop"); + assertEquals(0, transform.getErrors()); + blocker.get(5, TimeUnit.SECONDS); + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } finally { + releaseQueue.countDown(); + executor.shutdownNow(); + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void finishWaitsForSlowChatOutputBeforeClosingOutput() throws Exception { + // Regression: the 30 s drain cap used to apply on the normal end-of-input path too, so finish() + // abandoned a slow in-flight chat, cleared the facades and closed the output, dropping the row. + // finish() must wait without a cap so the row is still delivered. + final var chatStarted = new CountDownLatch(1); + final var releaseChat = new CountDownLatch(1); + try (final var facades = + mockConstruction( + LanguageModelFacade.class, + (facade, context) -> { + final var aiMessage = mock(AiMessage.class); + when(aiMessage.text()).thenReturn("answer"); + final var aiResponse = mock(ChatResponse.class); + when(aiResponse.aiMessage()).thenReturn(aiMessage); + when(facade.inputToChatMessages(any())).thenReturn(List.of()); + when(facade.chat(any())) + .thenAnswer( + invocation -> { + chatStarted.countDown(); + assertTrue(releaseChat.await(5, TimeUnit.SECONDS)); + return aiResponse; + }); + })) { + final var transform = transform(1); + transform.getMeta().setMock(false); + assertTrue(transform.processRow()); + assertTrue(chatStarted.await(5, TimeUnit.SECONDS)); + final var eof = Executors.newSingleThreadExecutor(); + try { + final var completion = + eof.submit( + () -> { + try { + assertFalse(transform.processRow()); + } catch (HopException e) { + throw new AssertionError(e); + } + }); + // The end-of-input path must keep waiting for the in-flight chat rather than abandoning it + // after a short drain cap and closing the output. + Thread.sleep(100); + assertFalse(completion.isDone()); + releaseChat.countDown(); + completion.get(5, TimeUnit.SECONDS); + final var row = transform.getOutputRowSets().getFirst().getRow(); + assertEquals("answer", row[transform.getData().indexOfOutput]); + assertEquals(0, transform.getErrors()); + transform.dispose(); + } finally { + releaseChat.countDown(); + eof.shutdownNow(); + } + } + } +} diff --git a/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java b/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java index 3ee791ab522..6952ae93619 100644 --- a/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java +++ b/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java @@ -26,8 +26,7 @@ import static org.apache.hop.pipeline.transforms.stanford.nlp.simple.ParsedSentence.Builder.parsedSentenceBuilder; import java.util.List; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.ExecutorService; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopRuntimeException; import org.apache.hop.core.exception.HopTransformException; @@ -44,8 +43,16 @@ public class StanfordSimpleNlp extends BaseTransform PKG = StanfordSimpleNlp.class; // For Translator private final int cores = Runtime.getRuntime().availableProcessors(); - private final ForkJoinPool executor = commonPool(); - private final AtomicInteger jobs = new AtomicInteger(0); + private final ExecutorService executor = commonPool(); + + /** How long draining waits for accepted asynchronous work before abandoning it. */ + private static final long DRAIN_TIMEOUT_MILLIS = 30_000L; + + // Guarded by lock, including reservation before submission. + private int jobs; + private Throwable failure; + // Guards against surfacing the same asynchronous failure twice (finish/checkFailure and dispose). + private boolean failureReported; private final int maxJobs = cores * 100; private final Object lock = new Object(); private ParsedSentence.Builder sentenceBuilder; @@ -60,22 +67,105 @@ public StanfordSimpleNlp( super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); } - private void finish() { - // Wait until all jobs are finished. + private void awaitJobs() { + awaitJobs(DRAIN_TIMEOUT_MILLIS); + } + + private void awaitJobs(final long timeoutMillis) { + boolean interrupted = false; + final long deadline = System.currentTimeMillis() + timeoutMillis; synchronized (lock) { - while (jobs.get() > 0) { + long remaining = timeoutMillis; + while (jobs > 0 && remaining > 0) { try { - lock.wait(); + lock.wait(remaining); } catch (InterruptedException e) { - throw new HopRuntimeException(e); + // Accepted work should finish before output is closed or resources are disposed, but an + // interrupted waiter must not hold up the transform or the pipeline. + interrupted = true; } + remaining = deadline - System.currentTimeMillis(); + } + if (jobs > 0) { + // A worker is stuck and no provider timeout applies. Do not block disposal forever: log and + // abandon the remaining jobs. + logBasic( + "Abandoning " + + jobs + + " asynchronous job(s) that did not finish within " + + timeoutMillis + + " ms"); } } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + /** + * Wait for every accepted job to complete on the normal end-of-input path. Unlike {@link + * #awaitJobs(long)} there is no timeout: rows that are merely slow must not be lost because the + * output is closed while workers still run. A stop (see {@link #stopRunning()}) breaks the wait. + */ + private void awaitJobsUncapped() { + boolean interrupted = false; + synchronized (lock) { + while (jobs > 0 && !isStopped()) { + try { + lock.wait(1_000L); + } catch (InterruptedException e) { + // Accepted work should finish before output is closed, but an interrupted waiter must + // not hold up the transform or the pipeline. + interrupted = true; + } + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private void checkFailure() throws HopException { + synchronized (lock) { + if (failure != null) { + failureReported = true; + throw new HopException("Asynchronous processing failed", failure); + } + } + } + + private void finish() throws HopException { + awaitJobsUncapped(); + checkFailure(); setOutputDone(); } + @Override + public void dispose() { + // The common pool is shared. Drain our work, but never shut the pool down. + awaitJobs(); + final boolean shouldLog; + synchronized (lock) { + // Only log a failure we have not already surfaced through the run loop, so it is not logged + // twice; the error accounting below still applies in either case. + shouldLog = failure != null && !failureReported; + if (shouldLog) { + failureReported = true; + } + } + if (failure != null) { + if (shouldLog) { + logError("Asynchronous processing failed", failure); + } + setErrors(1); + stopAll(); + } + super.dispose(); + } + @Override public boolean processRow() throws HopException { + checkFailure(); Object[] r = getRow(); // Get row from input rowset & set row busy! if (r == null) { // no more input to be expected... finish(); @@ -101,12 +191,8 @@ public boolean processRow() throws HopException { sentenceBuilder = parsedSentenceBuilder().includePartOfSpeech(meta.isIncludePartOfSpeech()); } // End If first - boolean sendToErrorRow = false; - String errorMessage = null; - - process(r); - try { + process(r); if (isRowLevel()) { logRowlevel( BaseMessages.getString( @@ -115,29 +201,15 @@ public boolean processRow() throws HopException { getLinesRead() + " : " + getInputRowMeta().getString(r))); } } catch (Exception e) { - if (getTransformMeta().isDoingErrorHandling()) { - sendToErrorRow = true; - errorMessage = e.toString(); - } else { - logError( - BaseMessages.getString(PKG, "StanfordSimpleNlp.ErrorInTransformRunning") - + e.getMessage()); - setErrors(1); - stopAll(); - setOutputDone(); // signal end to receiver(s) - return false; - } - if (sendToErrorRow) { - // Simply add this row to the error row - putError( - getInputRowMeta(), r, 1, errorMessage, meta.getCorpusField(), "StanfordSimpleNlp001"); - } + // An infrastructure/worker failure must not be attributed to the current input row. + checkFailure(); + handleRowError(r, e); } return true; } - private void process(Object[] inputRow) { + private void process(Object[] inputRow) throws HopException { String corpus = trim((String) inputRow[data.indexOfCorpusField]); if (isBlank(corpus)) { return; @@ -149,31 +221,63 @@ private void process(Object[] inputRow) { } } - private void processAsync(String corpus, Object[] inputRow) { + private void handleRowError(Object[] row, Exception e) throws HopException { + if (!getTransformMeta().isDoingErrorHandling()) { + throw new HopException("Error processing row", e); + } + // Error output has the same serialization requirement as normal output. + synchronized (this) { + putError( + getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(), "StanfordSimpleNlp001"); + } + } + + private void processAsync(String text, Object[] inputRow) throws HopException { synchronized (lock) { - while (jobs.get() > maxJobs) { + while (jobs >= maxJobs && failure == null) { try { lock.wait(); } catch (InterruptedException e) { - throw new HopRuntimeException(e); + Thread.currentThread().interrupt(); + throw new HopException("Interrupted waiting for async processing", e); } } + checkFailure(); + jobs++; } - executor.submit( - () -> { - try { - processSync(corpus, inputRow); - jobs.decrementAndGet(); - synchronized (lock) { - lock.notifyAll(); + try { + executor.submit( + () -> { + try { + try { + processSync(text, inputRow); + } catch (Exception e) { + handleRowError(inputRow, e); + } + } catch (Throwable e) { + synchronized (lock) { + if (failure == null) { + failure = e; + } + } + } finally { + synchronized (lock) { + jobs--; + lock.notifyAll(); + } } - } catch (Exception e) { - logError(e.getMessage(), e); - } - }); - - jobs.incrementAndGet(); + }); + } catch (RuntimeException | Error e) { + synchronized (lock) { + jobs--; + if (failure == null) { + failure = e; + } + lock.notifyAll(); + } + checkFailure(); + } } private void processSync(String corpus, Object[] inputRow) { diff --git a/plugins/transforms/stanfordnlp/src/test/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlpAsyncTest.java b/plugins/transforms/stanfordnlp/src/test/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlpAsyncTest.java new file mode 100644 index 00000000000..d1d55236183 --- /dev/null +++ b/plugins/transforms/stanfordnlp/src/test/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlpAsyncTest.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.stanford.nlp.simple; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(10) +class StanfordSimpleNlpAsyncTest { + private StanfordSimpleNlp transform; + private TransformMeta transformMeta; + private final Object[] row = {"

text

"}; + + @BeforeEach + void setUp() throws Exception { + org.apache.hop.core.logging.HopLogStore.init(); + StanfordSimpleNlpMeta meta = new StanfordSimpleNlpMeta(); + meta.setCorpusField("input"); + meta.setParallelism(true); + StanfordSimpleNlpData data = new StanfordSimpleNlpData(); + data.outputRowMeta = new RowMeta(); + PipelineMeta pipelineMeta = new PipelineMeta(); + transformMeta = spy(new TransformMeta("test", meta)); + pipelineMeta.addTransform(transformMeta); + transform = spy(new StanfordSimpleNlp(transformMeta, meta, data, 0, pipelineMeta, null)); + transform.first = false; + data.indexOfCorpusField = 0; + meta.setOutputFieldNamePrefix(""); + for (int i = 0; i < data.sentenceFields.length; i++) { + data.indexOfMap.put(data.sentenceFields[i], i + 1); + } + ParsedSentence.Builder builder = mock(ParsedSentence.Builder.class); + ParsedSentence sentence = mock(ParsedSentence.class); + when(sentence.getInputRow()).thenReturn(row); + when(sentence.getSentenceText()).thenReturn("text"); + when(builder.buildSentences(any(), anyString())).thenReturn(List.of(sentence)); + setField("sentenceBuilder", builder); + + doReturn(row, (Object[]) null).when(transform).getRow(); + doNothing().when(transform).setOutputDone(); + doNothing().when(transform).putRow(any(), any()); + } + + @Test + void inlineCompletionDoesNotLeavePhantomJob() throws Exception { + setField("executor", new InlineExecutor()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform).putRow(any(), any()); + verify(transform).setOutputDone(); + } + + @Test + void workerExceptionIsVisibleAtEof() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new HopTransformException("worker failed"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + HopException failure = assertThrows(HopException.class, transform::processRow); + assertTrue(failure.getMessage().contains("Asynchronous processing failed")); + } + + @Test + void workerErrorAlsoReleasesReservation() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new AssertionError("worker error"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertThrows(HopException.class, transform::processRow); + // Disposal must not hang even when submit captured an Error in its Future. + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void rejectedSubmissionRollsBackReservation() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + executor.shutdown(); + setField("executor", executor); + assertThrows(HopException.class, transform::processRow); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void synchronousFailureUsesConfiguredErrorHandling() throws Exception { + setParallelism(false); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void asynchronousFailureUsesConfiguredErrorHandling() throws Exception { + setField("executor", new InlineExecutor()); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad async row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void failureWakesAdmissionAtCapacity() throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + setField("executor", workers); + setField("maxJobs", 1); + doReturn(row).when(transform).getRow(); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + throw new HopTransformException("worker failed while admission waits"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch admitting = new CountDownLatch(1); + Future nextRow = + caller.submit( + () -> { + admitting.countDown(); + return assertThrows(HopException.class, transform::processRow); + }); + assertTrue(admitting.await(5, TimeUnit.SECONDS)); + assertFalse(nextRow.isDone()); + release.countDown(); + nextRow.get(5, TimeUnit.SECONDS); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void eofDrainsAcceptedWorkBeforeOutputDone() throws Exception { + assertDrains(false); + } + + @Test + void disposalDrainsAcceptedWorkWithoutShuttingDownExecutor() throws Exception { + assertDrains(true); + } + + private void assertDrains(boolean dispose) throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger outputs = new AtomicInteger(); + try { + setField("executor", workers); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + outputs.incrementAndGet(); + return null; + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch draining = new CountDownLatch(1); + Future completion = + caller.submit( + () -> { + draining.countDown(); + if (dispose) { + transform.dispose(); + } else { + try { + assertFalse(transform.processRow()); + } catch (HopException e) { + throw new AssertionError(e); + } + } + }); + assertTrue(draining.await(5, TimeUnit.SECONDS)); + assertFalse(completion.isDone()); + release.countDown(); + completion.get(5, TimeUnit.SECONDS); + assertEquals(1, outputs.get()); + assertFalse(workers.isShutdown()); + if (!dispose) { + verify(transform).setOutputDone(); + } + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void eofWaitsForSlowInFlightJobAndDeliversRowBeforeOutputDone() throws Exception { + // Regression: the 30 s drain cap used to apply on the normal end-of-input path too, so finish() + // abandoned jobs still running after 30 s and dropped their rows. finish() must wait without a + // cap and only then close the output. + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger outputs = new AtomicInteger(); + try { + setField("executor", workers); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + outputs.incrementAndGet(); + return null; + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + // Run the EOF step on its own thread so we can observe that, while a job is still in flight, + // it keeps waiting instead of dropping the job and closing the output on a short drain cap. + CountDownLatch draining = new CountDownLatch(1); + Future completion = + caller.submit( + () -> { + draining.countDown(); + try { + assertFalse(transform.processRow()); + } catch (HopException e) { + throw new AssertionError(e); + } + }); + assertTrue(draining.await(5, TimeUnit.SECONDS)); + // The in-flight job is still running; the EOF path must not have returned (closed the output) + // yet, even though it has been waiting longer than any sub-second drain cap. + Thread.sleep(100); + assertFalse(completion.isDone()); + release.countDown(); + completion.get(5, TimeUnit.SECONDS); + assertEquals(1, outputs.get()); + verify(transform).setOutputDone(); + assertFalse(workers.isShutdown()); + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void eofHonoursStopToShortCircuitUncappedWait() throws Exception { + // Regression: finish() waits without a cap for in-flight jobs, so a stop must still break the + // wait and close the output, otherwise a never-completing job would hang the transform. + ExecutorService workers = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + setField("executor", workers); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + return null; + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + doReturn(true).when(transform).isStopped(); + assertFalse(transform.processRow()); + verify(transform).setOutputDone(); + } finally { + release.countDown(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private void setParallelism(boolean parallelism) { + ((StanfordSimpleNlpMeta) transformMeta.getTransform()).setParallelism(parallelism); + } + + private void setField(String name, Object value) throws Exception { + Field field = StanfordSimpleNlp.class.getDeclaredField(name); + field.setAccessible(true); + field.set(transform, value); + } + + private static final class InlineExecutor extends AbstractExecutorService { + @Override + public void execute(Runnable command) { + command.run(); + } + + @Override + public void shutdown() {} + + @Override + public List shutdownNow() { + return List.of(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return false; + } + } +}