From fcfff93dcca3a8e82d28675629a346522c2e9aeb Mon Sep 17 00:00:00 2001 From: Romain Manni-Bucau Date: Sat, 7 Nov 2026 16:51:42 +0100 Subject: [PATCH 1/2] Fixes #9399 : Fix local pipeline shutdown deadlock and reentrant stopping --- .../org/apache/hop/pipeline/Pipeline.java | 60 +++--- .../hop/pipeline/transform/BaseTransform.java | 37 ++-- .../org/apache/hop/pipeline/PipelineTest.java | 182 ++++++++++++++++++ .../pipeline/transform/BaseTransformTest.java | 63 ++++++ 4 files changed, 291 insertions(+), 51 deletions(-) diff --git a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java index 9079528ef3b..e68e1ac00e2 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java +++ b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java @@ -1396,19 +1396,18 @@ private void startThreadsInternal() throws HopException { + pipelineRunConfiguration.getName() + "'"); } + } - // If a transform fails with an error, we want to kill/stop the others - // too... - // - if (transform.getErrors() > 0) { + // If a transform fails with an error, we want to kill/stop the others + // too... + // + if (transform.getErrors() > 0) { - log.logMinimal(BaseMessages.getString(PKG, "Pipeline.Log.PipelineDetectedErrors")); - log.logMinimal( - BaseMessages.getString( - PKG, "Pipeline.Log.PipelineIsKillingTheOtherTransforms")); + log.logMinimal(BaseMessages.getString(PKG, "Pipeline.Log.PipelineDetectedErrors")); + log.logMinimal( + BaseMessages.getString(PKG, "Pipeline.Log.PipelineIsKillingTheOtherTransforms")); - killAllNoWait(); - } + killAllNoWait(); } }; @@ -1893,7 +1892,7 @@ private void killAllNoWait() { return; } - for (TransformMetaDataCombi sid : transforms) { + for (TransformMetaDataCombi sid : new ArrayList<>(transforms)) { ITransform transform = sid.transform; if (log.isDebug()) { @@ -1957,7 +1956,8 @@ public void safeStop() { if (transforms == null) { return; } - transforms.stream().filter(this::isInputTransform).forEach(combi -> stopTransform(combi, true)); + new ArrayList<>(transforms) + .stream().filter(this::isInputTransform).forEach(combi -> stopTransform(combi, true)); fireExecutionStoppedListeners(); } @@ -1970,27 +1970,29 @@ private boolean isInputTransform(TransformMetaDataCombi combi) { /** Stops all transforms from running, and alerts any registered listeners. */ @Override public void stopAll() { - if (isAlreadyStopped.get()) { + if (!isAlreadyStopped.compareAndSet(false, true)) { return; } - // A pipeline that never got as far as allocating its transforms can still be stopped: it just - // has nothing to stop. Bailing out here used to leave it without a terminal state. - if (transforms != null) { - transforms.forEach(combi -> stopTransform(combi, false)); - } - - // if it is stopped it is not paused, nor is it still preparing or initializing - setPaused(false); - setPreparing(false); - setInitializing(false); - setStopped(true); - isAlreadyStopped.set(true); - if (debugController != null) { - debugController.cancelWaiters(); + try { + // A pipeline that never got as far as allocating its transforms can still be stopped: it just + // has nothing to stop. Bailing out here used to leave it without a terminal state. + if (transforms != null) { + // Do not hold the synchronized list monitor while calling transform code. + new ArrayList<>(transforms).forEach(combi -> stopTransform(combi, false)); + } + } finally { + // A failing stop callback must not leave the pipeline without a terminal state or skip the + // stopped listeners: set them even when one of the transforms above throws. + setPaused(false); + setPreparing(false); + setInitializing(false); + setStopped(true); + if (debugController != null) { + debugController.cancelWaiters(); + } + fireExecutionStoppedListeners(); } - - fireExecutionStoppedListeners(); } public void stopTransform(TransformMetaDataCombi combi, boolean safeStop) { diff --git a/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java b/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java index 32bbf27739a..ec6695adc21 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java +++ b/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java @@ -3246,33 +3246,26 @@ public void setInternalVariables() { * @see org.apache.hop.pipeline.transform.ITransform#markStop() */ @Override - public synchronized void markStop() { - - // Only mark a transform as stopped once - // - if (!markStopped.get()) { - markStopped.set(true); - - Calendar cal = Calendar.getInstance(); - stopTime = cal.getTime(); - - // We're finally completely done with this transform. - // + public void markStop() { + synchronized (this) { + // Publish the terminal state once, but never call external listeners under this monitor. + // A listener can stop the pipeline and acquire another transform's monitor. + if (!markStopped.compareAndSet(false, true)) { + return; + } + stopTime = Calendar.getInstance().getTime(); setRunning(false); - - // Here we are completely done with the pipeline. - // Call all the attached listeners and notify the outside world that the transform has - // finished. - // - fireTransformFinishedListeners(); } + fireTransformFinishedListeners(); } - private synchronized void fireTransformFinishedListeners() { + private void fireTransformFinishedListeners() { + final List listeners; synchronized (transformFinishedListeners) { - for (ITransformFinishedListener transformListener : transformFinishedListeners) { - transformListener.transformFinished(pipeline, transformMeta, this); - } + listeners = new ArrayList<>(transformFinishedListeners); + } + for (ITransformFinishedListener transformListener : listeners) { + transformListener.transformFinished(pipeline, transformMeta, this); } } diff --git a/engine/src/test/java/org/apache/hop/pipeline/PipelineTest.java b/engine/src/test/java/org/apache/hop/pipeline/PipelineTest.java index 84ac02e2744..a21ab971a0d 100644 --- a/engine/src/test/java/org/apache/hop/pipeline/PipelineTest.java +++ b/engine/src/test/java/org/apache/hop/pipeline/PipelineTest.java @@ -17,14 +17,17 @@ package org.apache.hop.pipeline; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -33,7 +36,9 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.hop.core.Const; import org.apache.hop.core.HopEnvironment; @@ -48,11 +53,13 @@ import org.apache.hop.core.plugins.IPlugin; import org.apache.hop.core.variables.Variables; import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.pipeline.engine.EngineComponent.ComponentExecutionStatus; import org.apache.hop.pipeline.engine.IPipelineEngine; import org.apache.hop.pipeline.engines.local.LocalPipelineEngine; import org.apache.hop.pipeline.transform.ITransform; import org.apache.hop.pipeline.transform.ITransformData; import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transform.TransformMetaDataCombi; import org.apache.hop.pipeline.transforms.dummy.DummyMeta; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -631,4 +638,179 @@ void stopAllOnlyFiresTheStoppedListenersOnce() { assertEquals(1, stopped[0]); } + + @Test + void stopAllCallbacksCanAccessTransformsFromAnotherThread() throws Exception { + assertStopCallbackCanAccessTransforms(false); + } + + @Test + void safeStopCallbacksCanAccessTransformsFromAnotherThread() throws Exception { + assertStopCallbackCanAccessTransforms(true); + } + + private void assertStopCallbackCanAccessTransforms(final boolean safeStop) throws Exception { + final var pipeline = new LocalPipelineEngine(new PipelineMeta()); + pipeline.setLogChannel(mock(ILogChannel.class)); + final var combi = new TransformMetaDataCombi(); + combi.transformMeta = new TransformMeta("input", new DummyMeta()); + combi.transform = mock(ITransform.class); + combi.data = mock(ITransformData.class); + pipeline.getPipelineMeta().addTransform(combi.transformMeta); + pipeline.setTransforms(Collections.synchronizedList(new ArrayList<>(List.of(combi)))); + final var reader = Executors.newSingleThreadExecutor(); + try { + doAnswer( + invocation -> { + // size() acquires the synchronized list's monitor. The callback must not hold it + // while waiting for another thread to inspect the pipeline. + final var size = reader.submit(() -> pipeline.getTransforms().size()); + assertEquals(1, assertDoesNotThrow(() -> size.get(5, TimeUnit.SECONDS))); + return null; + }) + .when(combi.transform) + .stopRunning(); + + if (safeStop) { + pipeline.safeStop(); + } else { + pipeline.stopAll(); + } + + verify(combi.transform).stopRunning(); + } finally { + reader.shutdownNow(); + assertTrue(reader.awaitTermination(5, TimeUnit.SECONDS), "Transform reader did not finish"); + } + } + + @Test + void stopAllDoesNotReenterTransformStopCallbacks() throws Exception { + final var pipeline = new LocalPipelineEngine(new PipelineMeta()); + pipeline.setLogChannel(mock(ILogChannel.class)); + final var combi = new TransformMetaDataCombi(); + combi.transform = mock(ITransform.class); + combi.data = mock(ITransformData.class); + pipeline.setTransforms(Collections.synchronizedList(new ArrayList<>(List.of(combi)))); + final var callbacks = new AtomicInteger(); + final var notifications = new AtomicInteger(); + pipeline.addExecutionStoppedListener(p -> notifications.incrementAndGet()); + doAnswer( + invocation -> { + // Bound a broken implementation to one recursive call instead of overflowing the + // stack: the observable callback count must still be exactly one. + if (callbacks.incrementAndGet() == 1) { + pipeline.stopAll(); + } + return null; + }) + .when(combi.transform) + .stopRunning(); + + pipeline.stopAll(); + + assertEquals(1, callbacks.get()); + assertEquals(1, notifications.get()); + assertTrue(pipeline.isStopped()); + } + + @Test + void stopAllStillReachesTerminalStateWhenAStopCallbackThrows() throws Exception { + final var pipeline = new LocalPipelineEngine(new PipelineMeta()); + pipeline.setLogChannel(mock(ILogChannel.class)); + final var combi = new TransformMetaDataCombi(); + combi.transform = mock(ITransform.class); + combi.data = mock(ITransformData.class); + pipeline.setTransforms(Collections.synchronizedList(new ArrayList<>(List.of(combi)))); + final var notifications = new AtomicInteger(); + pipeline.addExecutionStoppedListener(p -> notifications.incrementAndGet()); + // A misbehaving transform throws while it is being stopped: the pipeline must still reach its + // terminal stopped state and notify the stopped listeners exactly once. + doThrow(new RuntimeException("boom")).when(combi.transform).stopRunning(); + + pipeline.stopAll(); + + assertEquals(1, notifications.get()); + assertTrue(pipeline.isStopped()); + assertFalse(pipeline.isPaused()); + } + + @Test + void concurrentStopAllCallsNotifyStoppedListenersOnce() throws Exception { + final var pipeline = new LocalPipelineEngine(new PipelineMeta()); + pipeline.setLogChannel(mock(ILogChannel.class)); + final var combi = new TransformMetaDataCombi(); + combi.transform = mock(ITransform.class); + combi.data = mock(ITransformData.class); + // Isolate the one-shot guard from the synchronized list monitor: both calls can reach the + // callback in a broken implementation, without serializing on list iteration. + pipeline.setTransforms(new ArrayList<>(List.of(combi))); + final var callbackEntered = new CountDownLatch(1); + final var releaseCallback = new CountDownLatch(1); + final var callbacks = new AtomicInteger(); + final var notifications = new AtomicInteger(); + pipeline.addExecutionStoppedListener(p -> notifications.incrementAndGet()); + doAnswer( + invocation -> { + if (callbacks.incrementAndGet() == 1) { + callbackEntered.countDown(); + assertTrue(releaseCallback.await(10, TimeUnit.SECONDS), "Callback not released"); + } + return null; + }) + .when(combi.transform) + .stopRunning(); + final var callers = Executors.newFixedThreadPool(2); + try { + final var first = callers.submit(pipeline::stopAll); + assertTrue(callbackEntered.await(5, TimeUnit.SECONDS), "First stop did not reach callback"); + final var second = callers.submit(pipeline::stopAll); + // Keep the first stop inside the callback until the competing stop has returned. + second.get(5, TimeUnit.SECONDS); + releaseCallback.countDown(); + first.get(5, TimeUnit.SECONDS); + + assertEquals(1, notifications.get()); + assertEquals(1, callbacks.get()); + assertTrue(pipeline.isStopped()); + } finally { + releaseCallback.countDown(); + callers.shutdownNow(); + assertTrue(callers.awaitTermination(5, TimeUnit.SECONDS), "Stop callers did not finish"); + } + } + + @Test + void safeStopOnlyStopsInputTransforms() throws Exception { + final var meta = new PipelineMeta(); + final var input = new TransformMetaDataCombi(); + input.transformMeta = new TransformMeta("input", new DummyMeta()); + input.transform = mock(ITransform.class); + input.data = mock(ITransformData.class); + final var downstream = new TransformMetaDataCombi(); + downstream.transformMeta = new TransformMeta("downstream", new DummyMeta()); + downstream.transform = mock(ITransform.class); + downstream.data = mock(ITransformData.class); + meta.addTransform(input.transformMeta); + meta.addTransform(downstream.transformMeta); + meta.addPipelineHop(new PipelineHopMeta(input.transformMeta, downstream.transformMeta)); + final var pipeline = new LocalPipelineEngine(meta); + pipeline.setLogChannel(mock(ILogChannel.class)); + pipeline.setTransforms( + Collections.synchronizedList(new ArrayList<>(List.of(input, downstream)))); + + pipeline.safeStop(); + + verify(input.transform).setStopped(true); + verify(input.transform).setSafeStopped(true); + verify(input.transform).stopRunning(); + verify(input.transform).setOutputDone(); + verify(input.data).setStatus(ComponentExecutionStatus.STATUS_STOPPED); + verify(downstream.transform, never()).setStopped(true); + verify(downstream.transform, never()).setSafeStopped(true); + verify(downstream.transform, never()).stopRunning(); + verify(downstream.transform, never()).setOutputDone(); + verify(downstream.data, never()).setStatus(ComponentExecutionStatus.STATUS_STOPPED); + assertFalse(pipeline.isStopped(), "Downstream transforms must be allowed to drain"); + } } diff --git a/engine/src/test/java/org/apache/hop/pipeline/transform/BaseTransformTest.java b/engine/src/test/java/org/apache/hop/pipeline/transform/BaseTransformTest.java index 9945778eb0c..7f3ee3ab293 100644 --- a/engine/src/test/java/org/apache/hop/pipeline/transform/BaseTransformTest.java +++ b/engine/src/test/java/org/apache/hop/pipeline/transform/BaseTransformTest.java @@ -42,7 +42,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hop.core.BlockingRowSet; import org.apache.hop.core.Const; import org.apache.hop.core.IRowSet; @@ -204,6 +207,66 @@ public void run() { } } + @Test + void finishedListenerCanAcquireTransformAndListenerMonitorsFromAnotherThread() { + final var transform = + new BaseTransform<>( + mockHelper.transformMeta, + mockHelper.iTransformMeta, + mockHelper.iTransformData, + 0, + mockHelper.pipelineMeta, + mockHelper.pipeline); + final var calls = new AtomicInteger(); + transform.addTransformFinishedListener( + (pipeline, meta, finished) -> { + final var acquired = + CompletableFuture.runAsync( + () -> { + synchronized (transform) { + // stopRunning() implementations also acquire this monitor. + synchronized (transform.getTransformFinishedListeners()) { + calls.incrementAndGet(); + } + } + }); + try { + acquired.get(5, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError("Completion callback retained a lifecycle monitor", e); + } + }); + transform.markStart(); + transform.markStop(); + transform.markStop(); + assertEquals(1, calls.get()); + } + + @Test + void concurrentMarkStopNotifiesOnceAndAllowsListenerRegistration() throws Exception { + final var transform = + new BaseTransform<>( + mockHelper.transformMeta, + mockHelper.iTransformMeta, + mockHelper.iTransformData, + 0, + mockHelper.pipelineMeta, + mockHelper.pipeline); + final var calls = new AtomicInteger(); + transform.addTransformFinishedListener( + (pipeline, meta, finished) -> { + calls.incrementAndGet(); + // Adding a listener during notification must not invalidate iteration. + transform.addTransformFinishedListener((p, m, t) -> calls.addAndGet(100)); + transform.markStop(); + }); + transform.markStart(); + final var first = CompletableFuture.runAsync(transform::markStop); + final var second = CompletableFuture.runAsync(transform::markStop); + CompletableFuture.allOf(first, second).get(5, TimeUnit.SECONDS); + assertEquals(1, calls.get()); + } + @Test void resultFilesMapIsSafeForConcurrentModification() throws Exception { final BaseTransform transform = From 24c058c0c0be64eb9f1d4cc637190eea80000e37 Mon Sep 17 00:00:00 2001 From: Romain Manni-Bucau Date: Tue, 10 Nov 2026 23:54:49 +0100 Subject: [PATCH 2/2] fix: add integration test for 0049 execute sql row error stops pipeline, fixes #9399 Add an integration test covering the error-triggered local pipeline shutdown that this PR fixes (#9401). The pipeline runs Table Input (3000 rows) to Filter rows to Execute SQL row with no error hop; one INSERT (id 1500) violates a CHECK constraint so the SQL error must stop the whole local pipeline instead of hanging. The workflow runs the setup script then the pipeline, and finishes with a failure result without hanging. --- ...9-execute-sql-row-error-stops-pipeline.hpl | 139 ++++++++++++++++++ ...9-execute-sql-row-error-stops-pipeline.hwf | 116 +++++++++++++++ ...9-execute-sql-row-error-stops-pipeline.sql | 33 +++++ 3 files changed, 288 insertions(+) create mode 100644 integration-tests/database/0049-execute-sql-row-error-stops-pipeline.hpl create mode 100644 integration-tests/database/main-0049-execute-sql-row-error-stops-pipeline.hwf create mode 100644 integration-tests/database/scripts/0049-execute-sql-row-error-stops-pipeline.sql diff --git a/integration-tests/database/0049-execute-sql-row-error-stops-pipeline.hpl b/integration-tests/database/0049-execute-sql-row-error-stops-pipeline.hpl new file mode 100644 index 00000000000..1c60b3b6d73 --- /dev/null +++ b/integration-tests/database/0049-execute-sql-row-error-stops-pipeline.hpl @@ -0,0 +1,139 @@ + + + + + 0049-execute-sql-row-error-stops-pipeline + Y + Table Input (a few thousand rows) to Filter rows to Execute SQL row. One INSERT (id 1500) violates the target CHECK constraint and there is no error hop, so the SQL error must stop the whole local pipeline (error-triggered shutdown) instead of hanging. + + + Normal + 0 + + + N + 1000 + 100 + - + 2026/11/10 12:00:00.000 + - + 2026/11/10 12:00:00.000 + + + + + + Table Input + Filter rows + Y + + + Filter rows + Execute SQL row + Y + + + + Table Input + TableInput + + Y + + 1 + + none + + + unit-test-db + SELECT id, name, sql FROM public.it0049_execute_sql_row_stops_source + 0 + + N + N + + + 80 + 80 + + + + Filter rows + FilterRows + + Y + + 1 + + none + + + Execute SQL row + + + + N + id + <= + + + constant + Integer + 2000 + -1 + 0 + N + ####0;-####0 + + + + + + 240 + 80 + + + + Execute SQL row + ExecSqlRow + + Y + + 1 + + none + + + 2 + unit-test-db + sql + + + + + N + Y + + + 400 + 80 + + + + \ No newline at end of file diff --git a/integration-tests/database/main-0049-execute-sql-row-error-stops-pipeline.hwf b/integration-tests/database/main-0049-execute-sql-row-error-stops-pipeline.hwf new file mode 100644 index 00000000000..9b4229c3b31 --- /dev/null +++ b/integration-tests/database/main-0049-execute-sql-row-error-stops-pipeline.hwf @@ -0,0 +1,116 @@ + + + + main-0049-execute-sql-row-error-stops-pipeline + Y + Runs 0049-execute-sql-row-error-stops-pipeline, which must fail on one row and stop the local pipeline instead of hanging. The workflow expects the failure: the pipeline action returns a failed result, so the whole workflow must finish with a failure result and must not hang. + + + - + 2026/11/10 12:00:00.000 + - + 2026/11/10 12:00:00.000 + + + + + Start + + SPECIAL + + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 64 + 64 + + + + Create tables + + SQL + + + F + T + ${PROJECT_HOME}/scripts/0049-execute-sql-row-error-stops-pipeline.sql + F + unit-test-db + N + 224 + 64 + + + + 0049-execute-sql-row-error-stops-pipeline + + PIPELINE + + ${PROJECT_HOME}/0049-execute-sql-row-error-stops-pipeline.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 416 + 64 + + + + + + Start + Create tables + Y + Y + Y + + + Create tables + 0049-execute-sql-row-error-stops-pipeline + Y + Y + N + + + + + + \ No newline at end of file diff --git a/integration-tests/database/scripts/0049-execute-sql-row-error-stops-pipeline.sql b/integration-tests/database/scripts/0049-execute-sql-row-error-stops-pipeline.sql new file mode 100644 index 00000000000..87da2f0635e --- /dev/null +++ b/integration-tests/database/scripts/0049-execute-sql-row-error-stops-pipeline.sql @@ -0,0 +1,33 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +DROP TABLE IF EXISTS public.it0049_execute_sql_row_stops_target; +CREATE TABLE public.it0049_execute_sql_row_stops_target ( + id BIGINT PRIMARY KEY, + name VARCHAR(5), + CHECK (id <> 1500) +); + +DROP TABLE IF EXISTS public.it0049_execute_sql_row_stops_source; +CREATE TABLE public.it0049_execute_sql_row_stops_source ( + id BIGINT, + name VARCHAR(5), + sql VARCHAR(200) +); +INSERT INTO public.it0049_execute_sql_row_stops_source (id, name, sql) +SELECT g, 'a', 'INSERT INTO public.it0049_execute_sql_row_stops_target VALUES (' || g || ', ''a'')' +FROM generate_series(1, 3000) g; \ No newline at end of file