diff --git a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java index 9079528ef3b..0591af8b4bd 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java +++ b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java @@ -1396,19 +1396,18 @@ private void startThreadsInternal() throws HopException { + pipelineRunConfiguration.getName() + "'"); } + } - // If a transform fails with an error, we want to kill/stop the others - // too... - // - if (transform.getErrors() > 0) { + // If a transform fails with an error, we want to kill/stop the others + // too... + // + if (transform.getErrors() > 0) { - log.logMinimal(BaseMessages.getString(PKG, "Pipeline.Log.PipelineDetectedErrors")); - log.logMinimal( - BaseMessages.getString( - PKG, "Pipeline.Log.PipelineIsKillingTheOtherTransforms")); + log.logMinimal(BaseMessages.getString(PKG, "Pipeline.Log.PipelineDetectedErrors")); + log.logMinimal( + BaseMessages.getString(PKG, "Pipeline.Log.PipelineIsKillingTheOtherTransforms")); - killAllNoWait(); - } + killAllNoWait(); } }; @@ -1893,7 +1892,7 @@ private void killAllNoWait() { return; } - for (TransformMetaDataCombi sid : transforms) { + for (TransformMetaDataCombi sid : new ArrayList<>(transforms)) { ITransform transform = sid.transform; if (log.isDebug()) { @@ -1957,7 +1956,8 @@ public void safeStop() { if (transforms == null) { return; } - transforms.stream().filter(this::isInputTransform).forEach(combi -> stopTransform(combi, true)); + new ArrayList<>(transforms) + .stream().filter(this::isInputTransform).forEach(combi -> stopTransform(combi, true)); fireExecutionStoppedListeners(); } @@ -1970,14 +1970,15 @@ private boolean isInputTransform(TransformMetaDataCombi combi) { /** Stops all transforms from running, and alerts any registered listeners. */ @Override public void stopAll() { - if (isAlreadyStopped.get()) { + if (!isAlreadyStopped.compareAndSet(false, true)) { return; } // A pipeline that never got as far as allocating its transforms can still be stopped: it just // has nothing to stop. Bailing out here used to leave it without a terminal state. if (transforms != null) { - transforms.forEach(combi -> stopTransform(combi, false)); + // Do not hold the synchronized list monitor while calling transform code. + new ArrayList<>(transforms).forEach(combi -> stopTransform(combi, false)); } // if it is stopped it is not paused, nor is it still preparing or initializing @@ -1985,7 +1986,6 @@ public void stopAll() { setPreparing(false); setInitializing(false); setStopped(true); - isAlreadyStopped.set(true); if (debugController != null) { debugController.cancelWaiters(); } diff --git a/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java b/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java index 32bbf27739a..ec6695adc21 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java +++ b/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java @@ -3246,33 +3246,26 @@ public void setInternalVariables() { * @see org.apache.hop.pipeline.transform.ITransform#markStop() */ @Override - public synchronized void markStop() { - - // Only mark a transform as stopped once - // - if (!markStopped.get()) { - markStopped.set(true); - - Calendar cal = Calendar.getInstance(); - stopTime = cal.getTime(); - - // We're finally completely done with this transform. - // + public void markStop() { + synchronized (this) { + // Publish the terminal state once, but never call external listeners under this monitor. + // A listener can stop the pipeline and acquire another transform's monitor. + if (!markStopped.compareAndSet(false, true)) { + return; + } + stopTime = Calendar.getInstance().getTime(); setRunning(false); - - // Here we are completely done with the pipeline. - // Call all the attached listeners and notify the outside world that the transform has - // finished. - // - fireTransformFinishedListeners(); } + fireTransformFinishedListeners(); } - private synchronized void fireTransformFinishedListeners() { + private void fireTransformFinishedListeners() { + final List listeners; synchronized (transformFinishedListeners) { - for (ITransformFinishedListener transformListener : transformFinishedListeners) { - transformListener.transformFinished(pipeline, transformMeta, this); - } + listeners = new ArrayList<>(transformFinishedListeners); + } + for (ITransformFinishedListener transformListener : listeners) { + transformListener.transformFinished(pipeline, transformMeta, this); } } diff --git a/engine/src/test/java/org/apache/hop/pipeline/PipelineTest.java b/engine/src/test/java/org/apache/hop/pipeline/PipelineTest.java index 84ac02e2744..ea286b18d5a 100644 --- a/engine/src/test/java/org/apache/hop/pipeline/PipelineTest.java +++ b/engine/src/test/java/org/apache/hop/pipeline/PipelineTest.java @@ -17,14 +17,17 @@ package org.apache.hop.pipeline; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -33,7 +36,9 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.hop.core.Const; import org.apache.hop.core.HopEnvironment; @@ -48,11 +53,13 @@ import org.apache.hop.core.plugins.IPlugin; import org.apache.hop.core.variables.Variables; import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.pipeline.engine.EngineComponent.ComponentExecutionStatus; import org.apache.hop.pipeline.engine.IPipelineEngine; import org.apache.hop.pipeline.engines.local.LocalPipelineEngine; import org.apache.hop.pipeline.transform.ITransform; import org.apache.hop.pipeline.transform.ITransformData; import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transform.TransformMetaDataCombi; import org.apache.hop.pipeline.transforms.dummy.DummyMeta; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -631,4 +638,158 @@ void stopAllOnlyFiresTheStoppedListenersOnce() { assertEquals(1, stopped[0]); } + + @Test + void stopAllCallbacksCanAccessTransformsFromAnotherThread() throws Exception { + assertStopCallbackCanAccessTransforms(false); + } + + @Test + void safeStopCallbacksCanAccessTransformsFromAnotherThread() throws Exception { + assertStopCallbackCanAccessTransforms(true); + } + + private void assertStopCallbackCanAccessTransforms(final boolean safeStop) throws Exception { + final var pipeline = new LocalPipelineEngine(new PipelineMeta()); + pipeline.setLogChannel(mock(ILogChannel.class)); + final var combi = new TransformMetaDataCombi(); + combi.transformMeta = new TransformMeta("input", new DummyMeta()); + combi.transform = mock(ITransform.class); + combi.data = mock(ITransformData.class); + pipeline.getPipelineMeta().addTransform(combi.transformMeta); + pipeline.setTransforms(Collections.synchronizedList(new ArrayList<>(List.of(combi)))); + final var reader = Executors.newSingleThreadExecutor(); + try { + doAnswer( + invocation -> { + // size() acquires the synchronized list's monitor. The callback must not hold it + // while waiting for another thread to inspect the pipeline. + final var size = reader.submit(() -> pipeline.getTransforms().size()); + assertEquals(1, assertDoesNotThrow(() -> size.get(5, TimeUnit.SECONDS))); + return null; + }) + .when(combi.transform) + .stopRunning(); + + if (safeStop) { + pipeline.safeStop(); + } else { + pipeline.stopAll(); + } + + verify(combi.transform).stopRunning(); + } finally { + reader.shutdownNow(); + assertTrue(reader.awaitTermination(5, TimeUnit.SECONDS), "Transform reader did not finish"); + } + } + + @Test + void stopAllDoesNotReenterTransformStopCallbacks() throws Exception { + final var pipeline = new LocalPipelineEngine(new PipelineMeta()); + pipeline.setLogChannel(mock(ILogChannel.class)); + final var combi = new TransformMetaDataCombi(); + combi.transform = mock(ITransform.class); + combi.data = mock(ITransformData.class); + pipeline.setTransforms(Collections.synchronizedList(new ArrayList<>(List.of(combi)))); + final var callbacks = new AtomicInteger(); + final var notifications = new AtomicInteger(); + pipeline.addExecutionStoppedListener(p -> notifications.incrementAndGet()); + doAnswer( + invocation -> { + // Bound a broken implementation to one recursive call instead of overflowing the + // stack: the observable callback count must still be exactly one. + if (callbacks.incrementAndGet() == 1) { + pipeline.stopAll(); + } + return null; + }) + .when(combi.transform) + .stopRunning(); + + pipeline.stopAll(); + + assertEquals(1, callbacks.get()); + assertEquals(1, notifications.get()); + assertTrue(pipeline.isStopped()); + } + + @Test + void concurrentStopAllCallsNotifyStoppedListenersOnce() throws Exception { + final var pipeline = new LocalPipelineEngine(new PipelineMeta()); + pipeline.setLogChannel(mock(ILogChannel.class)); + final var combi = new TransformMetaDataCombi(); + combi.transform = mock(ITransform.class); + combi.data = mock(ITransformData.class); + // Isolate the one-shot guard from the synchronized list monitor: both calls can reach the + // callback in a broken implementation, without serializing on list iteration. + pipeline.setTransforms(new ArrayList<>(List.of(combi))); + final var callbackEntered = new CountDownLatch(1); + final var releaseCallback = new CountDownLatch(1); + final var callbacks = new AtomicInteger(); + final var notifications = new AtomicInteger(); + pipeline.addExecutionStoppedListener(p -> notifications.incrementAndGet()); + doAnswer( + invocation -> { + if (callbacks.incrementAndGet() == 1) { + callbackEntered.countDown(); + assertTrue(releaseCallback.await(10, TimeUnit.SECONDS), "Callback not released"); + } + return null; + }) + .when(combi.transform) + .stopRunning(); + final var callers = Executors.newFixedThreadPool(2); + try { + final var first = callers.submit(pipeline::stopAll); + assertTrue(callbackEntered.await(5, TimeUnit.SECONDS), "First stop did not reach callback"); + final var second = callers.submit(pipeline::stopAll); + // Keep the first stop inside the callback until the competing stop has returned. + second.get(5, TimeUnit.SECONDS); + releaseCallback.countDown(); + first.get(5, TimeUnit.SECONDS); + + assertEquals(1, notifications.get()); + assertEquals(1, callbacks.get()); + assertTrue(pipeline.isStopped()); + } finally { + releaseCallback.countDown(); + callers.shutdownNow(); + assertTrue(callers.awaitTermination(5, TimeUnit.SECONDS), "Stop callers did not finish"); + } + } + + @Test + void safeStopOnlyStopsInputTransforms() throws Exception { + final var meta = new PipelineMeta(); + final var input = new TransformMetaDataCombi(); + input.transformMeta = new TransformMeta("input", new DummyMeta()); + input.transform = mock(ITransform.class); + input.data = mock(ITransformData.class); + final var downstream = new TransformMetaDataCombi(); + downstream.transformMeta = new TransformMeta("downstream", new DummyMeta()); + downstream.transform = mock(ITransform.class); + downstream.data = mock(ITransformData.class); + meta.addTransform(input.transformMeta); + meta.addTransform(downstream.transformMeta); + meta.addPipelineHop(new PipelineHopMeta(input.transformMeta, downstream.transformMeta)); + final var pipeline = new LocalPipelineEngine(meta); + pipeline.setLogChannel(mock(ILogChannel.class)); + pipeline.setTransforms( + Collections.synchronizedList(new ArrayList<>(List.of(input, downstream)))); + + pipeline.safeStop(); + + verify(input.transform).setStopped(true); + verify(input.transform).setSafeStopped(true); + verify(input.transform).stopRunning(); + verify(input.transform).setOutputDone(); + verify(input.data).setStatus(ComponentExecutionStatus.STATUS_STOPPED); + verify(downstream.transform, never()).setStopped(true); + verify(downstream.transform, never()).setSafeStopped(true); + verify(downstream.transform, never()).stopRunning(); + verify(downstream.transform, never()).setOutputDone(); + verify(downstream.data, never()).setStatus(ComponentExecutionStatus.STATUS_STOPPED); + assertFalse(pipeline.isStopped(), "Downstream transforms must be allowed to drain"); + } } diff --git a/engine/src/test/java/org/apache/hop/pipeline/transform/BaseTransformTest.java b/engine/src/test/java/org/apache/hop/pipeline/transform/BaseTransformTest.java index 9945778eb0c..7f3ee3ab293 100644 --- a/engine/src/test/java/org/apache/hop/pipeline/transform/BaseTransformTest.java +++ b/engine/src/test/java/org/apache/hop/pipeline/transform/BaseTransformTest.java @@ -42,7 +42,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hop.core.BlockingRowSet; import org.apache.hop.core.Const; import org.apache.hop.core.IRowSet; @@ -204,6 +207,66 @@ public void run() { } } + @Test + void finishedListenerCanAcquireTransformAndListenerMonitorsFromAnotherThread() { + final var transform = + new BaseTransform<>( + mockHelper.transformMeta, + mockHelper.iTransformMeta, + mockHelper.iTransformData, + 0, + mockHelper.pipelineMeta, + mockHelper.pipeline); + final var calls = new AtomicInteger(); + transform.addTransformFinishedListener( + (pipeline, meta, finished) -> { + final var acquired = + CompletableFuture.runAsync( + () -> { + synchronized (transform) { + // stopRunning() implementations also acquire this monitor. + synchronized (transform.getTransformFinishedListeners()) { + calls.incrementAndGet(); + } + } + }); + try { + acquired.get(5, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError("Completion callback retained a lifecycle monitor", e); + } + }); + transform.markStart(); + transform.markStop(); + transform.markStop(); + assertEquals(1, calls.get()); + } + + @Test + void concurrentMarkStopNotifiesOnceAndAllowsListenerRegistration() throws Exception { + final var transform = + new BaseTransform<>( + mockHelper.transformMeta, + mockHelper.iTransformMeta, + mockHelper.iTransformData, + 0, + mockHelper.pipelineMeta, + mockHelper.pipeline); + final var calls = new AtomicInteger(); + transform.addTransformFinishedListener( + (pipeline, meta, finished) -> { + calls.incrementAndGet(); + // Adding a listener during notification must not invalidate iteration. + transform.addTransformFinishedListener((p, m, t) -> calls.addAndGet(100)); + transform.markStop(); + }); + transform.markStart(); + final var first = CompletableFuture.runAsync(transform::markStop); + final var second = CompletableFuture.runAsync(transform::markStop); + CompletableFuture.allOf(first, second).get(5, TimeUnit.SECONDS); + assertEquals(1, calls.get()); + } + @Test void resultFilesMapIsSafeForConcurrentModification() throws Exception { final BaseTransform transform = diff --git a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java index be1f3a5b679..c72d2427255 100644 --- a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java +++ b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java @@ -31,9 +31,7 @@ import java.util.Objects; import java.util.SortedMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.exception.HopRuntimeException; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.i18n.BaseMessages; @@ -59,30 +57,63 @@ public DetectLanguage( private final int cores = Runtime.getRuntime().availableProcessors(); private final int maxJobs = cores * 100; private final ExecutorService executor = commonPool(); - private final AtomicInteger jobs = new AtomicInteger(0); + // Guarded by lock, including reservation before submission. + private int jobs; + private Throwable failure; private final Object lock = new Object(); private final LanguageDetector detector = fromAllSpokenLanguages().withPreloadedLanguageModels().build(); - private void finish() { - // Wait until all jobs are finished. + private void awaitJobs() { + boolean interrupted = false; synchronized (lock) { - while (jobs.get() > 0) { + while (jobs > 0) { try { lock.wait(); } catch (InterruptedException e) { - throw new HopRuntimeException(e); + // Accepted work must finish before output is closed or resources are disposed. + interrupted = true; } } } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private void checkFailure() throws HopException { + synchronized (lock) { + if (failure != null) { + throw new HopException("Asynchronous processing failed", failure); + } + } + } + + private void finish() throws HopException { + awaitJobs(); detector.unloadLanguageModels(); + checkFailure(); setOutputDone(); } + @Override + public void dispose() { + // The common pool is shared. Drain our work, but never shut the pool down. + awaitJobs(); + detector.unloadLanguageModels(); + if (failure != null) { + logError("Asynchronous processing failed", failure); + setErrors(1); + stopAll(); + } + super.dispose(); + } + @Override public boolean processRow() throws HopException { + checkFailure(); Object[] r = getRow(); // Get row from input rowset & set row busy! if (r == null) { // no more input to be expected... finish(); @@ -106,12 +137,8 @@ public boolean processRow() throws HopException { cacheIndexPositions(); } // End If first - boolean sendToErrorRow = false; - String errorMessage = null; - - process(r); - try { + process(r); if (isRowLevel()) { logRowlevel( BaseMessages.getString( @@ -120,27 +147,15 @@ public boolean processRow() throws HopException { getLinesRead() + " : " + getInputRowMeta().getString(r))); } } catch (Exception e) { - if (getTransformMeta().isDoingErrorHandling()) { - sendToErrorRow = true; - errorMessage = e.toString(); - } else { - logError( - BaseMessages.getString(PKG, "DetectLanguage.ErrorInTransformRunning") + e.getMessage()); - setErrors(1); - stopAll(); - setOutputDone(); // signal end to receiver(s) - return false; - } - if (sendToErrorRow) { - // Simply add this row to the error row - putError(getInputRowMeta(), r, 1, errorMessage, meta.getCorpusField(), "DetectLanguage001"); - } + // An infrastructure/worker failure must not be attributed to the current input row. + checkFailure(); + handleRowError(r, e); } return true; } - private void process(Object[] inputRow) throws HopTransformException { + private void process(Object[] inputRow) throws HopException { String corpus = trim((String) inputRow[data.indexOfCorpusField]); if (isBlank(corpus)) { return; @@ -153,32 +168,62 @@ private void process(Object[] inputRow) throws HopTransformException { } } - private void processAsync(String corpus, Object[] inputRow) { + private void handleRowError(Object[] row, Exception e) throws HopException { + if (!getTransformMeta().isDoingErrorHandling()) { + throw new HopException("Error processing row", e); + } + // Error output has the same serialization requirement as normal output. + synchronized (this) { + putError(getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(), "DetectLanguage001"); + } + } + + private void processAsync(String text, Object[] inputRow) throws HopException { synchronized (lock) { - while (jobs.get() > maxJobs) { + while (jobs >= maxJobs && failure == null) { try { lock.wait(); } catch (InterruptedException e) { - throw new HopRuntimeException(e); + Thread.currentThread().interrupt(); + throw new HopException("Interrupted waiting for async processing", e); } } + checkFailure(); + jobs++; } - executor.submit( - () -> { - try { - processSync(corpus, inputRow); - jobs.decrementAndGet(); - - synchronized (lock) { - lock.notifyAll(); + try { + executor.submit( + () -> { + try { + try { + processSync(text, inputRow); + } catch (Exception e) { + handleRowError(inputRow, e); + } + } catch (Throwable e) { + synchronized (lock) { + if (failure == null) { + failure = e; + } + } + } finally { + synchronized (lock) { + jobs--; + lock.notifyAll(); + } } - } catch (Exception e) { - logError(e.getMessage(), e); - } - }); - - jobs.incrementAndGet(); + }); + } catch (RuntimeException | Error e) { + synchronized (lock) { + jobs--; + if (failure == null) { + failure = e; + } + lock.notifyAll(); + } + checkFailure(); + } } @SuppressWarnings( diff --git a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java new file mode 100644 index 00000000000..b808a69011e --- /dev/null +++ b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.language; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.RETURNS_SELF; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.github.pemistahl.lingua.api.LanguageDetector; +import com.github.pemistahl.lingua.api.LanguageDetectorBuilder; +import java.lang.reflect.Field; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.MockedStatic; + +@Timeout(10) +class DetectLanguageAsyncTest { + private DetectLanguage transform; + private TransformMeta transformMeta; + private final Object[] row = {"

text

"}; + + @BeforeEach + void setUp() throws Exception { + org.apache.hop.core.logging.HopLogStore.init(); + DetectLanguageMeta meta = new DetectLanguageMeta(); + meta.setCorpusField("input"); + meta.setParallelism(true); + DetectLanguageData data = new DetectLanguageData(); + data.outputRowMeta = new RowMeta(); + PipelineMeta pipelineMeta = new PipelineMeta(); + transformMeta = spy(new TransformMeta("test", meta)); + pipelineMeta.addTransform(transformMeta); + try (MockedStatic factory = + mockStatic(LanguageDetectorBuilder.class)) { + LanguageDetectorBuilder builder = mock(LanguageDetectorBuilder.class, RETURNS_SELF); + LanguageDetector detector = mock(LanguageDetector.class); + factory.when(LanguageDetectorBuilder::fromAllSpokenLanguages).thenReturn(builder); + when(builder.build()).thenReturn(detector); + when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new TreeMap<>()); + transform = spy(new DetectLanguage(transformMeta, meta, data, 0, pipelineMeta, null)); + } + transform.first = false; + data.indexOfCorpusField = 0; + data.indexOfDetectedLanguage = 1; + data.indexOfDetectedLanguageConfidence = 2; + + doReturn(row, (Object[]) null).when(transform).getRow(); + doNothing().when(transform).setOutputDone(); + doNothing().when(transform).putRow(any(), any()); + } + + @Test + void inlineCompletionDoesNotLeavePhantomJob() throws Exception { + setField("executor", new InlineExecutor()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform).putRow(any(), any()); + verify(transform).setOutputDone(); + } + + @Test + void workerExceptionIsVisibleAtEof() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new HopTransformException("worker failed"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + HopException failure = assertThrows(HopException.class, transform::processRow); + assertTrue(failure.getMessage().contains("Asynchronous processing failed")); + } + + @Test + void workerErrorAlsoReleasesReservation() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new AssertionError("worker error"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertThrows(HopException.class, transform::processRow); + // Disposal must not hang even when submit captured an Error in its Future. + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void rejectedSubmissionRollsBackReservation() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + executor.shutdown(); + setField("executor", executor); + assertThrows(HopException.class, transform::processRow); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void synchronousFailureUsesConfiguredErrorHandling() throws Exception { + setParallelism(false); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void asynchronousFailureUsesConfiguredErrorHandling() throws Exception { + setField("executor", new InlineExecutor()); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad async row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void failureWakesAdmissionAtCapacity() throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + setField("executor", workers); + setField("maxJobs", 1); + doReturn(row).when(transform).getRow(); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + throw new HopTransformException("worker failed while admission waits"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch admitting = new CountDownLatch(1); + Future nextRow = + caller.submit( + () -> { + admitting.countDown(); + return assertThrows(HopException.class, transform::processRow); + }); + assertTrue(admitting.await(5, TimeUnit.SECONDS)); + assertFalse(nextRow.isDone()); + release.countDown(); + nextRow.get(5, TimeUnit.SECONDS); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void eofDrainsAcceptedWorkBeforeOutputDone() throws Exception { + assertDrains(false); + } + + @Test + void disposalDrainsAcceptedWorkWithoutShuttingDownExecutor() throws Exception { + assertDrains(true); + } + + private void assertDrains(boolean dispose) throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger outputs = new AtomicInteger(); + try { + setField("executor", workers); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + outputs.incrementAndGet(); + return null; + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch draining = new CountDownLatch(1); + Future completion = + caller.submit( + () -> { + draining.countDown(); + if (dispose) { + transform.dispose(); + } else { + try { + assertFalse(transform.processRow()); + } catch (HopException e) { + throw new AssertionError(e); + } + } + }); + assertTrue(draining.await(5, TimeUnit.SECONDS)); + assertFalse(completion.isDone()); + release.countDown(); + completion.get(5, TimeUnit.SECONDS); + assertEquals(1, outputs.get()); + assertFalse(workers.isShutdown()); + if (!dispose) { + verify(transform).setOutputDone(); + } + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private void setParallelism(boolean parallelism) { + ((DetectLanguageMeta) transformMeta.getTransform()).setParallelism(parallelism); + } + + private void setField(String name, Object value) throws Exception { + Field field = DetectLanguage.class.getDeclaredField(name); + field.setAccessible(true); + field.set(transform, value); + } + + private static final class InlineExecutor extends AbstractExecutorService { + @Override + public void execute(Runnable command) { + command.run(); + } + + @Override + public void shutdown() {} + + @Override + public List shutdownNow() { + return List.of(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return false; + } + } +} diff --git a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java index a8590192563..b9d1aeb1691 100644 --- a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java +++ b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java @@ -18,16 +18,23 @@ package org.apache.hop.pipeline.transforms.execprocess; import java.io.BufferedReader; +import java.io.Closeable; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.CountDownLatch; +import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.StringUtils; import org.apache.hop.core.Const; import org.apache.hop.core.exception.HopException; @@ -44,8 +51,7 @@ public class ExecProcess extends BaseTransform private static final Class PKG = ExecProcessMeta.class; public static final String CONST_EXEC_PROCESS_EXCEPTION_COULD_NOT_FIND_FIELD = "ExecProcess.Exception.CouldNotFindField"; - private boolean killing; - private CountDownLatch waitForLatch; + private final AtomicReference activeExecution = new AtomicReference<>(); public ExecProcess( TransformMeta transformMeta, @@ -95,13 +101,19 @@ public boolean processRow() throws HopException { cmdArray.add(Const.NVL(argString, "")); } - execProcess(cmdArray.toArray(new String[0]), processResult); + if (!execProcess(cmdArray.toArray(new String[0]), processResult)) { + setOutputDone(); + return false; + } } else { if (isDetailed() && !hasBalancedQuotes(processString)) { logDetailed( BaseMessages.getString(PKG, "ExecProcess.Log.UnbalancedQuotes", processString)); } - execProcess(tokenizeCommandLine(processString), processResult); + if (!execProcess(tokenizeCommandLine(processString), processResult)) { + setOutputDone(); + return false; + } } if (meta.isFailWhenNotSuccess() && processResult.getExitValue() != 0) { @@ -199,124 +211,152 @@ private void initializeAndLookup() throws HopException { @Override public void stopRunning() throws HopException { - if (waitForLatch != null) { - killing = true; - try { - waitForLatch.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new HopException("Interrupted exception while kill the process", e); - } + // A pipeline stop must not wait for a child process or a stream reader. In particular, a + // descendant can keep a pipe open after the direct child exits. + setStopped(true); + final var execution = activeExecution.get(); + if (execution != null) { + execution.cancelled.set(true); } } - private void execProcess(String[] process, ProcessResult processresult) throws HopException { - Process p = null; - waitForLatch = new CountDownLatch(1); + private boolean execProcess(final String[] command, final ProcessResult result) + throws HopException { + final var execution = new ProcessExecution(); + activeExecution.set(execution); + Process child = null; + boolean completed = false; try { - String errorMsg = null; - // execute process + if (isStopped()) { + return false; + } try { - p = data.runtime.exec(process); - } catch (Exception e) { - errorMsg = e.getMessage(); + child = data.runtime.exec(command); + } catch (IOException e) { + if (isStopped() || execution.cancelled.get()) { + return false; + } + result.setErrorStream(e.getMessage()); + return true; } - if (p == null) { - processresult.setErrorStream(errorMsg); - } else { - final Process child = p; - // Drain stdout/stderr while the process runs. Reading only after onExit() can deadlock: - // if the child fills either pipe buffer before exiting, it blocks on write forever. - CompletableFuture stdoutFuture = - CompletableFuture.supplyAsync( - () -> { - try { - return getOutputString( - new BufferedReader(new InputStreamReader(child.getInputStream()))); - } catch (IOException e) { - throw new CompletionException(e); - } - }); - CompletableFuture stderrFuture = - CompletableFuture.supplyAsync( - () -> { - try { - return getOutputString( - new BufferedReader(new InputStreamReader(child.getErrorStream()))); - } catch (IOException e) { - throw new CompletionException(e); - } - }); - - while (true) { - try { - if (child.waitFor(1, TimeUnit.SECONDS)) { - break; - } - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new HopException( - "Interrupted exception while running the process " + Arrays.toString(process) + "!", - ie); - } - if (killing) { - child.children().forEach(ProcessHandle::destroy); - if (child.isAlive()) { - child.destroy(); - } - try { - if (!child.waitFor(30, TimeUnit.SECONDS)) { - child.destroyForcibly(); - child.waitFor(); - } - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new HopException( - "Interrupted exception while running the process " - + Arrays.toString(process) - + "!", - ie); - } - logMinimal(BaseMessages.getString(PKG, "ExecProcess.AbortProcess")); - break; - } + if (isCancelled(execution)) { + return false; + } + final Process process = child; + // Owned daemon readers keep blocked pipes out of the shared common pool. They close their + // streams on normal completion; cancellation also arranges asynchronous stream cleanup. + final var stdout = execution.readers.submit(() -> readOutput(process.getInputStream())); + final var stderr = execution.readers.submit(() -> readOutput(process.getErrorStream())); + while (true) { + // Remember children while the parent is alive: after it exits, inherited pipes may + // still be held by descendants that are no longer discoverable through that parent. + try (final var descendants = process.descendants()) { + descendants.forEach(execution.descendants::add); } - - try { - processresult.setOutputStream(stdoutFuture.get()); - processresult.setErrorStream(stderrFuture.get()); - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof IOException ioe) { - throw ioe; - } - if (cause instanceof RuntimeException re) { - throw re; - } - throw new IOException(cause); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new HopException( - "Interrupted exception while running the process " + Arrays.toString(process) + "!", - ie); + if (isCancelled(execution)) { + return false; } - - processresult.setExitValue(child.exitValue()); + if (process.waitFor(100, TimeUnit.MILLISECONDS)) { + break; + } + } + final var output = awaitOutput(stdout, execution); + if (isCancelled(execution)) { + return false; + } + final var error = awaitOutput(stderr, execution); + if (isCancelled(execution)) { + return false; } - } catch (IOException ioe) { - throw new HopException( - "IO exception while running the process " + Arrays.toString(process) + "!", ioe); - } catch (Exception e) { - throw new HopException(e); + result.setOutputStream(output); + result.setErrorStream(error); + result.setExitValue(process.exitValue()); + completed = true; + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new HopException("Interrupted while running process " + Arrays.toString(command), e); + } catch (ExecutionException e) { + throw new HopException("Unable to read process output", e.getCause()); } finally { - if (killing || waitForLatch.getCount() > 0) { - waitForLatch.countDown(); - killing = false; + activeExecution.compareAndSet(execution, null); + execution.readers.shutdownNow(); + if (child != null) { + cleanupProcess(child, execution, !completed); } - if (p != null) { - p.destroy(); + } + } + + private boolean isCancelled(final ProcessExecution execution) { + return isStopped() || execution.cancelled.get(); + } + + private String awaitOutput(final Future output, final ProcessExecution execution) + throws InterruptedException, ExecutionException { + while (!isCancelled(execution)) { + try { + return output.get(100, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + // Observe stop requests even after the process exits but a stream drainer is still active. } } + return null; + } + + private String readOutput(final InputStream stream) throws IOException { + try (final var reader = new BufferedReader(new InputStreamReader(stream))) { + return getOutputString(reader); + } + } + + private void cleanupProcess( + final Process process, final ProcessExecution execution, final boolean abort) { + // Process.destroy and stream.close may themselves block in native I/O or on a reader's + // monitor. Neither operation belongs on the pipeline's stopping or transform thread. + final var cleanup = + new Thread( + () -> { + try { + if (abort) { + try (final var descendants = process.descendants()) { + descendants.forEach(execution.descendants::add); + } + execution.descendants.forEach(ProcessHandle::destroyForcibly); + if (process.isAlive()) { + process.destroyForcibly(); + } + } + } finally { + closeProcessStream(process.getOutputStream()); + closeProcessStream(process.getInputStream()); + closeProcessStream(process.getErrorStream()); + } + }, + "hop-execprocess-cleanup"); + cleanup.setDaemon(true); + cleanup.start(); + } + + private void closeProcessStream(final Closeable stream) { + try { + stream.close(); + } catch (IOException e) { + logDebug("Unable to close process stream", e); + } + } + + private static final class ProcessExecution { + private final AtomicBoolean cancelled = new AtomicBoolean(); + // Owned by the executing thread, then handed off to the cleanup thread after execution. + private final Set descendants = new HashSet<>(); + private final ExecutorService readers = + Executors.newFixedThreadPool( + 2, + task -> { + final var thread = new Thread(task, "hop-execprocess-output"); + thread.setDaemon(true); + return thread; + }); } /** diff --git a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java index 90460541156..5b41afcbaa1 100644 --- a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java +++ b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java @@ -23,10 +23,20 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.util.Arrays; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import org.apache.hop.core.HopEnvironment; import org.apache.hop.core.IRowSet; import org.apache.hop.core.QueueRowSet; @@ -41,6 +51,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.condition.EnabledOnOs; import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.RegisterExtension; @@ -359,6 +370,160 @@ void init_requiresResultFieldName() throws HopException { assertFalse(transform.init()); } + @Test + @Timeout(10) + void stopWhileDrainingOutputDoesNotWaitForEndOfStream() throws Exception { + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + final var process = mock(Process.class); + final var stdout = new BlockingInputStream(); + when(process.getInputStream()).thenReturn(stdout); + when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream()); + when(process.waitFor(100, TimeUnit.MILLISECONDS)).thenReturn(true); + when(process.descendants()).thenAnswer(invocation -> Stream.empty()); + when(runtime.exec(any(String[].class))).thenReturn(process); + transform.getData().runtime = runtime; + final var executor = Executors.newSingleThreadExecutor(); + try { + final var running = executor.submit(transform::processRow); + assertTrue(stdout.reading.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + transform.stopRunning(); + assertFalse(running.get(5, TimeUnit.SECONDS)); + assertTrue(stdout.closed.await(5, TimeUnit.SECONDS)); + assertEquals(0, transform.getErrors()); + assertEquals(0, transform.getOutputRowSets().getFirst().size()); + } finally { + stdout.close(); + executor.shutdownNow(); + } + } + + @Test + @Timeout(10) + void stopDuringLaunchCancelsTheNewProcessWithoutBlockingStopper() throws Exception { + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + final var process = mock(Process.class); + final var launching = new CountDownLatch(1); + final var releaseLaunch = new CountDownLatch(1); + final var destroyed = new CountDownLatch(1); + when(runtime.exec(any(String[].class))) + .thenAnswer( + invocation -> { + launching.countDown(); + assertTrue(releaseLaunch.await(5, TimeUnit.SECONDS)); + return process; + }); + when(process.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream()); + when(process.isAlive()).thenReturn(true); + when(process.descendants()).thenAnswer(invocation -> Stream.empty()); + when(process.destroyForcibly()) + .thenAnswer( + invocation -> { + destroyed.countDown(); + return process; + }); + transform.getData().runtime = runtime; + final var executor = Executors.newSingleThreadExecutor(); + try { + final var running = executor.submit(transform::processRow); + assertTrue(launching.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + releaseLaunch.countDown(); + assertFalse(running.get(5, TimeUnit.SECONDS)); + assertTrue(destroyed.await(5, TimeUnit.SECONDS)); + assertEquals(0, transform.getErrors()); + } finally { + releaseLaunch.countDown(); + executor.shutdownNow(); + } + } + + @Test + @Timeout(10) + void cancellationRetainsChildrenObservedBeforeParentExit() throws Exception { + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + final var process = mock(Process.class); + final var descendant = mock(ProcessHandle.class); + final var destroyed = new CountDownLatch(1); + final var stdout = new BlockingInputStream(); + when(descendant.destroyForcibly()) + .thenAnswer( + invocation -> { + destroyed.countDown(); + return true; + }); + when(process.descendants()).thenReturn(Stream.of(descendant), Stream.empty()); + when(process.getInputStream()).thenReturn(stdout); + when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + when(process.getOutputStream()).thenReturn(new ByteArrayOutputStream()); + when(process.waitFor(100, TimeUnit.MILLISECONDS)).thenReturn(true); + when(runtime.exec(any(String[].class))).thenReturn(process); + transform.getData().runtime = runtime; + final var executor = Executors.newSingleThreadExecutor(); + try { + final var running = executor.submit(transform::processRow); + assertTrue(stdout.reading.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + assertFalse(running.get(5, TimeUnit.SECONDS)); + assertTrue(destroyed.await(5, TimeUnit.SECONDS)); + } finally { + stdout.close(); + executor.shutdownNow(); + } + } + + @Test + void stopBeforeLaunchDoesNotStartAProcess() throws Exception { + HopEnvironment.init(); + final var transform = + newTransform(metaForArgumentsMode(), inputRowMetaForArguments(), new Object[] {"test"}); + final var runtime = mock(Runtime.class); + transform.getData().runtime = runtime; + transform.stopRunning(); + assertFalse(transform.processRow()); + verify(runtime, never()).exec(any(String[].class)); + } + + private static final class BlockingInputStream extends InputStream { + private final CountDownLatch reading = new CountDownLatch(1); + private final CountDownLatch closed = new CountDownLatch(1); + + @Override + public int read() { + reading.countDown(); + boolean interrupted = false; + while (true) { + try { + closed.await(); + break; + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + return -1; + } + + @Override + public void close() { + closed.countDown(); + } + } + private ExecProcess newTransform(ExecProcessMeta meta, IRowMeta rowMeta, Object[] row) { TransformMeta tm = smh.transformMeta; when(tm.isDoingErrorHandling()).thenReturn(false); diff --git a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2Text.java b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2Text.java index e73b4a80bbc..08cb6f1ad4e 100644 --- a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2Text.java +++ b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2Text.java @@ -26,7 +26,6 @@ import static org.jsoup.Jsoup.parseBodyFragment; import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.core.row.IRowMeta; @@ -45,7 +44,9 @@ public class Html2Text extends BaseTransform { private final int cores = Runtime.getRuntime().availableProcessors(); private final int maxJobs = cores * 100; private final ExecutorService executor = commonPool(); - private final AtomicInteger jobs = new AtomicInteger(0); + // Guarded by lock, including reservation before submission. + private int jobs; + private Throwable failure; private final Object lock = new Object(); @@ -59,22 +60,52 @@ public Html2Text( super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); } - private void finish() throws HopException { - // Wait until all jobs are finished. + private void awaitJobs() { + boolean interrupted = false; synchronized (lock) { - while (jobs.get() > 0) { + while (jobs > 0) { try { lock.wait(); } catch (InterruptedException e) { - throw new HopException("Waiting for jobs to finished interrupted", e); + // Accepted work must finish before output is closed or resources are disposed. + interrupted = true; } } } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private void checkFailure() throws HopException { + synchronized (lock) { + if (failure != null) { + throw new HopException("Asynchronous processing failed", failure); + } + } + } + + private void finish() throws HopException { + awaitJobs(); + checkFailure(); setOutputDone(); } + @Override + public void dispose() { + // The common pool is shared. Drain our work, but never shut the pool down. + awaitJobs(); + if (failure != null) { + logError("Asynchronous processing failed", failure); + setErrors(1); + stopAll(); + } + super.dispose(); + } + @Override public boolean processRow() throws HopException { + checkFailure(); Object[] r = getRow(); // Get row from input rowset & set row busy! if (r == null) { // no more input to be expected... finish(); @@ -100,11 +131,8 @@ public boolean processRow() throws HopException { cacheIndexPositions(); } // End If first - String errorMessage = null; - - process(r); - try { + process(r); if (isRowLevel()) { logRowlevel( BaseMessages.getString( @@ -113,24 +141,15 @@ public boolean processRow() throws HopException { getLinesRead() + " : " + getInputRowMeta().getString(r))); } } catch (Exception e) { - if (getTransformMeta().isDoingErrorHandling()) { - errorMessage = e.toString(); - } else { - logError(BaseMessages.getString(PKG, "Html2Text.ErrorInTransformRunning") + e.getMessage()); - setErrors(1); - stopAll(); - setOutputDone(); // signal end to receiver(s) - return false; - } - - // Add this row to the error output of this transform - putError(getInputRowMeta(), r, 1, errorMessage, meta.getHtmlField(), "Html2Text001"); + // An infrastructure/worker failure must not be attributed to the current input row. + checkFailure(); + handleRowError(r, e); } return true; } - private void process(Object[] inputRow) throws HopTransformException { + private void process(Object[] inputRow) throws HopException { String html = trim((String) inputRow[data.indexOfHtmlField]); if (isBlank(html)) { return; @@ -143,32 +162,62 @@ private void process(Object[] inputRow) throws HopTransformException { } } - private void processAsync(String html, Object[] inputRow) throws HopTransformException { + private void handleRowError(Object[] row, Exception e) throws HopException { + if (!getTransformMeta().isDoingErrorHandling()) { + throw new HopException("Error processing row", e); + } + // Error output has the same serialization requirement as normal output. + synchronized (this) { + putError(getInputRowMeta(), row, 1, e.toString(), meta.getHtmlField(), "Html2Text001"); + } + } + + private void processAsync(String text, Object[] inputRow) throws HopException { synchronized (lock) { - while (jobs.get() > maxJobs) { + while (jobs >= maxJobs && failure == null) { try { lock.wait(); } catch (InterruptedException e) { - throw new HopTransformException("Error waiting for async processing lock release", e); + Thread.currentThread().interrupt(); + throw new HopException("Interrupted waiting for async processing", e); } } + checkFailure(); + jobs++; } - executor.submit( - () -> { - try { - processSync(html, inputRow); - jobs.decrementAndGet(); - - synchronized (lock) { - lock.notifyAll(); + try { + executor.submit( + () -> { + try { + try { + processSync(text, inputRow); + } catch (Exception e) { + handleRowError(inputRow, e); + } + } catch (Throwable e) { + synchronized (lock) { + if (failure == null) { + failure = e; + } + } + } finally { + synchronized (lock) { + jobs--; + lock.notifyAll(); + } } - } catch (Exception e) { - logError(e.getMessage(), e); - } - }); - - jobs.incrementAndGet(); + }); + } catch (RuntimeException | Error e) { + synchronized (lock) { + jobs--; + if (failure == null) { + failure = e; + } + lock.notifyAll(); + } + checkFailure(); + } } private void processSync(String html, Object[] inputRow) throws HopTransformException { diff --git a/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextAsyncTest.java b/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextAsyncTest.java new file mode 100644 index 00000000000..900959a62bb --- /dev/null +++ b/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextAsyncTest.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.html2text; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(10) +class Html2TextAsyncTest { + private Html2Text transform; + private TransformMeta transformMeta; + private final Object[] row = {"

text

"}; + + @BeforeEach + void setUp() throws Exception { + org.apache.hop.core.logging.HopLogStore.init(); + Html2TextMeta meta = new Html2TextMeta(); + meta.setHtmlField("input"); + meta.setParallelism(true); + Html2TextData data = new Html2TextData(); + data.outputRowMeta = new RowMeta(); + PipelineMeta pipelineMeta = new PipelineMeta(); + transformMeta = spy(new TransformMeta("test", meta)); + pipelineMeta.addTransform(transformMeta); + transform = spy(new Html2Text(transformMeta, meta, data, 0, pipelineMeta, null)); + transform.first = false; + meta.setOutputField("text"); + data.indexOfHtmlField = 0; + data.indexOfOutputField = 1; + + doReturn(row, (Object[]) null).when(transform).getRow(); + doNothing().when(transform).setOutputDone(); + doNothing().when(transform).putRow(any(), any()); + } + + @Test + void inlineCompletionDoesNotLeavePhantomJob() throws Exception { + setField("executor", new InlineExecutor()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform).putRow(any(), any()); + verify(transform).setOutputDone(); + } + + @Test + void workerExceptionIsVisibleAtEof() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new HopTransformException("worker failed"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + HopException failure = assertThrows(HopException.class, transform::processRow); + assertTrue(failure.getMessage().contains("Asynchronous processing failed")); + } + + @Test + void workerErrorAlsoReleasesReservation() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new AssertionError("worker error"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertThrows(HopException.class, transform::processRow); + // Disposal must not hang even when submit captured an Error in its Future. + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void rejectedSubmissionRollsBackReservation() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + executor.shutdown(); + setField("executor", executor); + assertThrows(HopException.class, transform::processRow); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void synchronousFailureUsesConfiguredErrorHandling() throws Exception { + setParallelism(false); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void asynchronousFailureUsesConfiguredErrorHandling() throws Exception { + setField("executor", new InlineExecutor()); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad async row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void failureWakesAdmissionAtCapacity() throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + setField("executor", workers); + setField("maxJobs", 1); + doReturn(row).when(transform).getRow(); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + throw new HopTransformException("worker failed while admission waits"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch admitting = new CountDownLatch(1); + Future nextRow = + caller.submit( + () -> { + admitting.countDown(); + return assertThrows(HopException.class, transform::processRow); + }); + assertTrue(admitting.await(5, TimeUnit.SECONDS)); + assertFalse(nextRow.isDone()); + release.countDown(); + nextRow.get(5, TimeUnit.SECONDS); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void eofDrainsAcceptedWorkBeforeOutputDone() throws Exception { + assertDrains(false); + } + + @Test + void disposalDrainsAcceptedWorkWithoutShuttingDownExecutor() throws Exception { + assertDrains(true); + } + + private void assertDrains(boolean dispose) throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger outputs = new AtomicInteger(); + try { + setField("executor", workers); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + outputs.incrementAndGet(); + return null; + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch draining = new CountDownLatch(1); + Future completion = + caller.submit( + () -> { + draining.countDown(); + if (dispose) { + transform.dispose(); + } else { + try { + assertFalse(transform.processRow()); + } catch (HopException e) { + throw new AssertionError(e); + } + } + }); + assertTrue(draining.await(5, TimeUnit.SECONDS)); + assertFalse(completion.isDone()); + release.countDown(); + completion.get(5, TimeUnit.SECONDS); + assertEquals(1, outputs.get()); + assertFalse(workers.isShutdown()); + if (!dispose) { + verify(transform).setOutputDone(); + } + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private void setParallelism(boolean parallelism) { + ((Html2TextMeta) transformMeta.getTransform()).setParallelism(parallelism); + } + + private void setField(String name, Object value) throws Exception { + Field field = Html2Text.class.getDeclaredField(name); + field.setAccessible(true); + field.set(transform, value); + } + + private static final class InlineExecutor extends AbstractExecutorService { + @Override + public void execute(Runnable command) { + command.run(); + } + + @Override + public void shutdown() {} + + @Override + public List shutdownNow() { + return List.of(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return false; + } + } +} diff --git a/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java b/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java index e6442fabbca..df0bdeb61db 100644 --- a/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java +++ b/plugins/transforms/languagemodelchat/src/main/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChat.java @@ -17,13 +17,11 @@ package org.apache.hop.pipeline.transforms.languagemodelchat; -import static java.lang.Runtime.getRuntime; import static java.lang.System.arraycopy; import static java.time.Duration.between; import static java.time.Instant.now; import static java.util.concurrent.Executors.newWorkStealingPool; import static java.util.concurrent.ForkJoinPool.commonPool; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.trim; @@ -37,7 +35,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopRuntimeException; @@ -62,8 +60,8 @@ public class LanguageModelChat extends BaseTransform facadeMap = new ConcurrentHashMap<>(); private LanguageModelChatMeta modelMeta; - private int parallelism = 1; - private ForkJoinPool executor; + private volatile ExecutorService executor; + private volatile boolean ownsExecutor; private final int cores = Runtime.getRuntime().availableProcessors(); private final AtomicInteger jobs = new AtomicInteger(0); private final int maxJobs = cores * 100; @@ -86,20 +84,13 @@ private void finish() { try { lock.wait(); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new HopRuntimeException(e); } } } - if (executor != null) { - try { - while (executor.hasQueuedSubmissions() || executor.getActiveThreadCount() > 0) { - SECONDS.sleep(5); - } - } catch (InterruptedException e) { - logError(e.getMessage(), e); - } - } + shutdownOwnedExecutor(); facadeMap.clear(); setOutputDone(); @@ -125,14 +116,11 @@ public boolean processRow() throws HopException { modelMeta = LanguageModelChatAiProviderSupport.resolve(meta, this, metadataProvider); facadeMap.put(new LanguageModel(modelMeta), new LanguageModelFacade(variables, modelMeta)); - int parallelism = - meta.getParallelism() <= 0 ? getRuntime().availableProcessors() : meta.getParallelism(); - if (parallelism != 0 && this.parallelism != parallelism || executor == null) { - if (executor != null) { - executor.shutdownNow(); - } - executor = (ForkJoinPool) newWorkStealingPool(parallelism); - } + ownsExecutor = meta.getParallelism() != 0; + executor = + ownsExecutor + ? newWorkStealingPool(meta.getParallelism() < 0 ? cores : meta.getParallelism()) + : commonPool(); if (isEmpty(meta.getInputField())) { throw new HopException( @@ -177,53 +165,105 @@ private void process(Object[] inputRow) { } synchronized (lock) { - while (jobs.get() > maxJobs) { + while (jobs.get() >= maxJobs && !isStopped()) { try { lock.wait(); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new HopRuntimeException(e); } } } - ExecutorService exec = meta.getParallelism() == 0 ? commonPool() : executor; - - exec.submit( - () -> { - try { - processSync(inputRow, message); - } catch (Exception e) { + if (isStopped()) { + return; + } + jobs.incrementAndGet(); + try { + executor.submit( + () -> { try { - if (getTransformMeta().isDoingErrorHandling()) { - String errorMessage = e.toString(); - putError( - getInputRowMeta(), - inputRow, - 1L, - errorMessage, - meta.getInputField(), - "LanguageModelChat001"); - } else { - logError(i18n("LanguageModelChat.ErrorInTransformRunning") + e.getMessage(), e); - setErrors(1L); - stopAll(); + processSync(inputRow, message); + } catch (Exception e) { + try { + if (getTransformMeta().isDoingErrorHandling()) { + String errorMessage = e.toString(); + putError( + getInputRowMeta(), + inputRow, + 1L, + errorMessage, + meta.getInputField(), + "LanguageModelChat001"); + } else { + logError(i18n("LanguageModelChat.ErrorInTransformRunning") + e.getMessage(), e); + setErrors(1L); + stopAll(); + } + } catch (HopTransformException ex) { + logError(e.getMessage(), e); + logError(ex.getMessage(), ex); + if (!getTransformMeta().isDoingErrorHandling()) { + setErrors(1L); + stopAll(); + } } - } catch (HopTransformException ex) { - logError(e.getMessage(), e); - logError(ex.getMessage(), ex); - if (!getTransformMeta().isDoingErrorHandling()) { - setErrors(1L); - stopAll(); - } - } - } finally { - jobs.decrementAndGet(); - synchronized (lock) { - lock.notifyAll(); + } finally { + jobFinished(); } - } - }); - jobs.incrementAndGet(); + }); + } catch (RejectedExecutionException e) { + jobFinished(); + if (!isStopped()) { + throw e; + } + } + } + + private void jobFinished() { + jobs.decrementAndGet(); + synchronized (lock) { + lock.notifyAll(); + } + } + + private void shutdownOwnedExecutor() { + if (ownsExecutor && executor != null) { + // Accepted tasks still drain. Never await this executor from a worker stopping the pipeline. + executor.shutdown(); + } + } + + @Override + public void stopRunning() throws HopException { + setStopped(true); + shutdownOwnedExecutor(); + synchronized (lock) { + lock.notifyAll(); + } + super.stopRunning(); + } + + @Override + public void dispose() { + shutdownOwnedExecutor(); + boolean interrupted = false; + synchronized (lock) { + while (jobs.get() > 0) { + try { + lock.wait(); + } catch (InterruptedException e) { + interrupted = true; + } + } + } + // The runner disposes after processRow returns, which need not have reached normal EOF after + // a stop or failure. Keep facades alive until every accepted worker has left processSync. + facadeMap.clear(); + super.dispose(); + if (interrupted) { + Thread.currentThread().interrupt(); + } } private void processSync(Object[] inputRow, String message) diff --git a/plugins/transforms/languagemodelchat/src/test/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChatLifecycleTest.java b/plugins/transforms/languagemodelchat/src/test/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChatLifecycleTest.java new file mode 100644 index 00000000000..fa2d2678635 --- /dev/null +++ b/plugins/transforms/languagemodelchat/src/test/java/org/apache/hop/pipeline/transforms/languagemodelchat/LanguageModelChatLifecycleTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.languagemodelchat; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinWorkerThread; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.QueueRowSet; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.logging.ILoggingObject; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.pipeline.transform.RowAdapter; +import org.apache.hop.pipeline.transforms.languagemodelchat.internals.LanguageModelFacade; +import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.extension.RegisterExtension; + +@Timeout(15) +class LanguageModelChatLifecycleTest { + @RegisterExtension + static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); + + private TransformMockHelper helper; + + @BeforeEach + void setUp() throws Exception { + helper = + new TransformMockHelper<>("chat", LanguageModelChatMeta.class, LanguageModelChatData.class); + when(helper.logChannelFactory.create(any(), any(ILoggingObject.class))) + .thenReturn(helper.iLogChannel); + when(helper.pipeline.isRunning()).thenReturn(true); + HopEnvironment.init(); + } + + @AfterEach + void cleanUp() { + helper.cleanUp(); + } + + private LanguageModelChat transform(final int parallelism) { + final var meta = new LanguageModelChatMeta(); + meta.setDefault(); + meta.setMock(true); + meta.setMockOutputValue("answer"); + meta.setParallelism(parallelism); + final var transform = + new LanguageModelChat( + helper.transformMeta, + meta, + new LanguageModelChatData(), + 0, + helper.pipelineMeta, + helper.pipeline); + final var rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaString("input")); + transform.setInputRowMeta(rowMeta); + final var input = helper.getMockInputRowSet(new Object[] {"question"}); + when(input.getRowMeta()).thenReturn(rowMeta); + transform.addRowSetToInputRowSets(input); + transform.addRowSetToOutputRowSets(new QueueRowSet()); + return transform; + } + + @Test + void ownedPoolDrainsMockOutputBeforeCompletion() throws Exception { + final var transform = transform(2); + final var pool = new AtomicReference(); + transform.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(final IRowMeta rowMeta, final Object[] row) { + pool.set(((ForkJoinWorkerThread) Thread.currentThread()).getPool()); + } + }); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + final var output = transform.getOutputRowSets().getFirst(); + assertEquals(1, output.size()); + final var row = output.getRow(); + assertEquals("MOCK", row[transform.getData().indexOfFinishReason]); + assertEquals("answer", row[transform.getData().indexOfOutput]); + assertTrue(output.isDone()); + transform.dispose(); + transform.dispose(); + assertTrue(pool.get().isShutdown()); + assertTrue(pool.get().awaitTermination(5, TimeUnit.SECONDS)); + assertEquals(0, transform.getErrors()); + } + + @Test + void providerExceptionStillProducesAnErrorResultRow() throws Exception { + try (final var facades = + mockConstruction( + LanguageModelFacade.class, + (facade, context) -> { + when(facade.inputToChatMessages(any())).thenReturn(List.of()); + when(facade.chat(any())).thenThrow(new IllegalStateException("provider unavailable")); + })) { + final var transform = transform(1); + transform.getMeta().setMock(false); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + final var row = transform.getOutputRowSets().getFirst().getRow(); + assertEquals("ERROR", row[transform.getData().indexOfFinishReason]); + assertEquals("provider unavailable", row[transform.getData().indexOfOutput]); + assertEquals(0, transform.getErrors()); + transform.dispose(); + } + } + + @Test + void rejectionDuringStopIsNotReportedAsARowFailure() throws Exception { + final var executor = mock(ExecutorService.class); + try (final var factory = mockStatic(Executors.class)) { + factory.when(() -> Executors.newWorkStealingPool(1)).thenReturn(executor); + final var transform = transform(1); + when(executor.submit(any(Runnable.class))) + .thenAnswer( + invocation -> { + transform.stopRunning(); + throw new RejectedExecutionException("stopping"); + }); + assertTrue(transform.processRow()); + transform.dispose(); + assertEquals(0, transform.getErrors()); + } + } + + @Test + void commonPoolRemainsUsableAfterCompletionAndDisposal() throws Exception { + final var transform = transform(0); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + transform.dispose(); + transform.stopRunning(); + assertFalse(ForkJoinPool.commonPool().isShutdown()); + assertEquals(42, ForkJoinPool.commonPool().submit(() -> 42).get(5, TimeUnit.SECONDS)); + } + + @Test + void workerFailureCanStopThePipelineWithoutWaitingForItself() throws Exception { + final var transform = transform(1); + final var stopping = new CountDownLatch(1); + doAnswer( + invocation -> { + transform.stopRunning(); + stopping.countDown(); + return null; + }) + .when(helper.pipeline) + .stopAll(); + transform.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(final IRowMeta rowMeta, final Object[] row) + throws HopTransformException { + throw new HopTransformException("output failed"); + } + }); + assertTrue(transform.processRow()); + assertTrue(stopping.await(5, TimeUnit.SECONDS)); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void disposeWaitsForAcceptedOutputWithoutHoldingItsMonitor() throws Exception { + final var transform = transform(1); + final var outputStarted = new CountDownLatch(1); + final var releaseOutput = new CountDownLatch(1); + final var workerFinished = new AtomicBoolean(); + transform.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(final IRowMeta rowMeta, final Object[] row) + throws HopTransformException { + outputStarted.countDown(); + try { + if (!releaseOutput.await(5, TimeUnit.SECONDS)) { + throw new HopTransformException("output wait timed out"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new HopTransformException(e); + } + workerFinished.set(true); + } + }); + final var disposer = Executors.newSingleThreadExecutor(); + try { + assertTrue(transform.processRow()); + assertTrue(outputStarted.await(5, TimeUnit.SECONDS)); + transform.stopRunning(); + final var disposed = + disposer.submit( + () -> { + transform.dispose(); + assertTrue( + workerFinished.get(), "dispose returned before accepted output completed"); + }); + assertFalse(disposed.isDone()); + releaseOutput.countDown(); + disposed.get(5, TimeUnit.SECONDS); + assertEquals(0, transform.getErrors()); + } finally { + releaseOutput.countDown(); + disposer.shutdownNow(); + } + } +} diff --git a/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java b/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java index f2d13dade68..213d596c7ac 100644 --- a/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java +++ b/plugins/transforms/stanfordnlp/src/main/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlp.java @@ -26,8 +26,7 @@ import static org.apache.hop.pipeline.transforms.stanford.nlp.simple.ParsedSentence.Builder.parsedSentenceBuilder; import java.util.List; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.ExecutorService; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopRuntimeException; import org.apache.hop.core.exception.HopTransformException; @@ -42,8 +41,10 @@ public class StanfordSimpleNlp extends BaseTransform PKG = StanfordSimpleNlp.class; // For Translator private final int cores = Runtime.getRuntime().availableProcessors(); - private final ForkJoinPool executor = commonPool(); - private final AtomicInteger jobs = new AtomicInteger(0); + private final ExecutorService executor = commonPool(); + // Guarded by lock, including reservation before submission. + private int jobs; + private Throwable failure; private final int maxJobs = cores * 100; private final Object lock = new Object(); private ParsedSentence.Builder sentenceBuilder; @@ -58,22 +59,52 @@ public StanfordSimpleNlp( super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); } - private void finish() { - // Wait until all jobs are finished. + private void awaitJobs() { + boolean interrupted = false; synchronized (lock) { - while (jobs.get() > 0) { + while (jobs > 0) { try { lock.wait(); } catch (InterruptedException e) { - throw new HopRuntimeException(e); + // Accepted work must finish before output is closed or resources are disposed. + interrupted = true; } } } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private void checkFailure() throws HopException { + synchronized (lock) { + if (failure != null) { + throw new HopException("Asynchronous processing failed", failure); + } + } + } + + private void finish() throws HopException { + awaitJobs(); + checkFailure(); setOutputDone(); } + @Override + public void dispose() { + // The common pool is shared. Drain our work, but never shut the pool down. + awaitJobs(); + if (failure != null) { + logError("Asynchronous processing failed", failure); + setErrors(1); + stopAll(); + } + super.dispose(); + } + @Override public boolean processRow() throws HopException { + checkFailure(); Object[] r = getRow(); // Get row from input rowset & set row busy! if (r == null) { // no more input to be expected... finish(); @@ -99,12 +130,8 @@ public boolean processRow() throws HopException { sentenceBuilder = parsedSentenceBuilder().includePartOfSpeech(meta.isIncludePartOfSpeech()); } // End If first - boolean sendToErrorRow = false; - String errorMessage = null; - - process(r); - try { + process(r); if (isRowLevel()) { logRowlevel( BaseMessages.getString( @@ -113,29 +140,15 @@ public boolean processRow() throws HopException { getLinesRead() + " : " + getInputRowMeta().getString(r))); } } catch (Exception e) { - if (getTransformMeta().isDoingErrorHandling()) { - sendToErrorRow = true; - errorMessage = e.toString(); - } else { - logError( - BaseMessages.getString(PKG, "StanfordSimpleNlp.ErrorInTransformRunning") - + e.getMessage()); - setErrors(1); - stopAll(); - setOutputDone(); // signal end to receiver(s) - return false; - } - if (sendToErrorRow) { - // Simply add this row to the error row - putError( - getInputRowMeta(), r, 1, errorMessage, meta.getCorpusField(), "StanfordSimpleNlp001"); - } + // An infrastructure/worker failure must not be attributed to the current input row. + checkFailure(); + handleRowError(r, e); } return true; } - private void process(Object[] inputRow) { + private void process(Object[] inputRow) throws HopException { String corpus = trim((String) inputRow[data.indexOfCorpusField]); if (isBlank(corpus)) { return; @@ -147,31 +160,63 @@ private void process(Object[] inputRow) { } } - private void processAsync(String corpus, Object[] inputRow) { + private void handleRowError(Object[] row, Exception e) throws HopException { + if (!getTransformMeta().isDoingErrorHandling()) { + throw new HopException("Error processing row", e); + } + // Error output has the same serialization requirement as normal output. + synchronized (this) { + putError( + getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(), "StanfordSimpleNlp001"); + } + } + + private void processAsync(String text, Object[] inputRow) throws HopException { synchronized (lock) { - while (jobs.get() > maxJobs) { + while (jobs >= maxJobs && failure == null) { try { lock.wait(); } catch (InterruptedException e) { - throw new HopRuntimeException(e); + Thread.currentThread().interrupt(); + throw new HopException("Interrupted waiting for async processing", e); } } + checkFailure(); + jobs++; } - executor.submit( - () -> { - try { - processSync(corpus, inputRow); - jobs.decrementAndGet(); - synchronized (lock) { - lock.notifyAll(); + try { + executor.submit( + () -> { + try { + try { + processSync(text, inputRow); + } catch (Exception e) { + handleRowError(inputRow, e); + } + } catch (Throwable e) { + synchronized (lock) { + if (failure == null) { + failure = e; + } + } + } finally { + synchronized (lock) { + jobs--; + lock.notifyAll(); + } } - } catch (Exception e) { - logError(e.getMessage(), e); - } - }); - - jobs.incrementAndGet(); + }); + } catch (RuntimeException | Error e) { + synchronized (lock) { + jobs--; + if (failure == null) { + failure = e; + } + lock.notifyAll(); + } + checkFailure(); + } } private void processSync(String corpus, Object[] inputRow) { diff --git a/plugins/transforms/stanfordnlp/src/test/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlpAsyncTest.java b/plugins/transforms/stanfordnlp/src/test/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlpAsyncTest.java new file mode 100644 index 00000000000..c6db75700ab --- /dev/null +++ b/plugins/transforms/stanfordnlp/src/test/java/org/apache/hop/pipeline/transforms/stanford/nlp/simple/StanfordSimpleNlpAsyncTest.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.stanford.nlp.simple; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(10) +class StanfordSimpleNlpAsyncTest { + private StanfordSimpleNlp transform; + private TransformMeta transformMeta; + private final Object[] row = {"

text

"}; + + @BeforeEach + void setUp() throws Exception { + org.apache.hop.core.logging.HopLogStore.init(); + StanfordSimpleNlpMeta meta = new StanfordSimpleNlpMeta(); + meta.setCorpusField("input"); + meta.setParallelism(true); + StanfordSimpleNlpData data = new StanfordSimpleNlpData(); + data.outputRowMeta = new RowMeta(); + PipelineMeta pipelineMeta = new PipelineMeta(); + transformMeta = spy(new TransformMeta("test", meta)); + pipelineMeta.addTransform(transformMeta); + transform = spy(new StanfordSimpleNlp(transformMeta, meta, data, 0, pipelineMeta, null)); + transform.first = false; + data.indexOfCorpusField = 0; + meta.setOutputFieldNamePrefix(""); + for (int i = 0; i < data.sentenceFields.length; i++) { + data.indexOfMap.put(data.sentenceFields[i], i + 1); + } + ParsedSentence.Builder builder = mock(ParsedSentence.Builder.class); + ParsedSentence sentence = mock(ParsedSentence.class); + when(sentence.getInputRow()).thenReturn(row); + when(sentence.getSentenceText()).thenReturn("text"); + when(builder.buildSentences(any(), anyString())).thenReturn(List.of(sentence)); + setField("sentenceBuilder", builder); + + doReturn(row, (Object[]) null).when(transform).getRow(); + doNothing().when(transform).setOutputDone(); + doNothing().when(transform).putRow(any(), any()); + } + + @Test + void inlineCompletionDoesNotLeavePhantomJob() throws Exception { + setField("executor", new InlineExecutor()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform).putRow(any(), any()); + verify(transform).setOutputDone(); + } + + @Test + void workerExceptionIsVisibleAtEof() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new HopTransformException("worker failed"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + HopException failure = assertThrows(HopException.class, transform::processRow); + assertTrue(failure.getMessage().contains("Asynchronous processing failed")); + } + + @Test + void workerErrorAlsoReleasesReservation() throws Exception { + setField("executor", new InlineExecutor()); + doAnswer( + invocation -> { + throw new AssertionError("worker error"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertThrows(HopException.class, transform::processRow); + // Disposal must not hang even when submit captured an Error in its Future. + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void rejectedSubmissionRollsBackReservation() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + executor.shutdown(); + setField("executor", executor); + assertThrows(HopException.class, transform::processRow); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } + + @Test + void synchronousFailureUsesConfiguredErrorHandling() throws Exception { + setParallelism(false); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void asynchronousFailureUsesConfiguredErrorHandling() throws Exception { + setField("executor", new InlineExecutor()); + doReturn(true).when(transformMeta).isDoingErrorHandling(); + doAnswer( + invocation -> { + throw new HopTransformException("bad async row"); + }) + .when(transform) + .putRow(any(), any()); + doNothing() + .when(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + assertTrue(transform.processRow()); + assertFalse(transform.processRow()); + verify(transform) + .putError( + any(), + any(), + org.mockito.ArgumentMatchers.anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void failureWakesAdmissionAtCapacity() throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + setField("executor", workers); + setField("maxJobs", 1); + doReturn(row).when(transform).getRow(); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + throw new HopTransformException("worker failed while admission waits"); + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch admitting = new CountDownLatch(1); + Future nextRow = + caller.submit( + () -> { + admitting.countDown(); + return assertThrows(HopException.class, transform::processRow); + }); + assertTrue(admitting.await(5, TimeUnit.SECONDS)); + assertFalse(nextRow.isDone()); + release.countDown(); + nextRow.get(5, TimeUnit.SECONDS); + doNothing().when(transform).stopAll(); + transform.dispose(); + assertEquals(1, transform.getErrors()); + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void eofDrainsAcceptedWorkBeforeOutputDone() throws Exception { + assertDrains(false); + } + + @Test + void disposalDrainsAcceptedWorkWithoutShuttingDownExecutor() throws Exception { + assertDrains(true); + } + + private void assertDrains(boolean dispose) throws Exception { + ExecutorService workers = Executors.newSingleThreadExecutor(); + ExecutorService caller = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger outputs = new AtomicInteger(); + try { + setField("executor", workers); + doAnswer( + invocation -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + outputs.incrementAndGet(); + return null; + }) + .when(transform) + .putRow(any(), any()); + assertTrue(transform.processRow()); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CountDownLatch draining = new CountDownLatch(1); + Future completion = + caller.submit( + () -> { + draining.countDown(); + if (dispose) { + transform.dispose(); + } else { + try { + assertFalse(transform.processRow()); + } catch (HopException e) { + throw new AssertionError(e); + } + } + }); + assertTrue(draining.await(5, TimeUnit.SECONDS)); + assertFalse(completion.isDone()); + release.countDown(); + completion.get(5, TimeUnit.SECONDS); + assertEquals(1, outputs.get()); + assertFalse(workers.isShutdown()); + if (!dispose) { + verify(transform).setOutputDone(); + } + } finally { + release.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private void setParallelism(boolean parallelism) { + ((StanfordSimpleNlpMeta) transformMeta.getTransform()).setParallelism(parallelism); + } + + private void setField(String name, Object value) throws Exception { + Field field = StanfordSimpleNlp.class.getDeclaredField(name); + field.setAccessible(true); + field.set(transform, value); + } + + private static final class InlineExecutor extends AbstractExecutorService { + @Override + public void execute(Runnable command) { + command.run(); + } + + @Override + public void shutdown() {} + + @Override + public List shutdownNow() { + return List.of(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return false; + } + } +}