Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions engine/src/main/java/org/apache/hop/pipeline/Pipeline.java
Original file line number Diff line number Diff line change
Expand Up @@ -1396,19 +1396,18 @@ private void startThreadsInternal() throws HopException {
+ pipelineRunConfiguration.getName()
+ "'");
}
}

// If a transform fails with an error, we want to kill/stop the others
// too...
//
if (transform.getErrors() > 0) {
// If a transform fails with an error, we want to kill/stop the others
// too...
//
if (transform.getErrors() > 0) {

log.logMinimal(BaseMessages.getString(PKG, "Pipeline.Log.PipelineDetectedErrors"));
log.logMinimal(
BaseMessages.getString(
PKG, "Pipeline.Log.PipelineIsKillingTheOtherTransforms"));
log.logMinimal(BaseMessages.getString(PKG, "Pipeline.Log.PipelineDetectedErrors"));
log.logMinimal(
BaseMessages.getString(PKG, "Pipeline.Log.PipelineIsKillingTheOtherTransforms"));

killAllNoWait();
}
killAllNoWait();
}
};

Expand Down Expand Up @@ -1893,7 +1892,7 @@ private void killAllNoWait() {
return;
}

for (TransformMetaDataCombi sid : transforms) {
for (TransformMetaDataCombi sid : new ArrayList<>(transforms)) {
ITransform transform = sid.transform;

if (log.isDebug()) {
Expand Down Expand Up @@ -1957,7 +1956,8 @@ public void safeStop() {
if (transforms == null) {
return;
}
transforms.stream().filter(this::isInputTransform).forEach(combi -> stopTransform(combi, true));
new ArrayList<>(transforms)
.stream().filter(this::isInputTransform).forEach(combi -> stopTransform(combi, true));

fireExecutionStoppedListeners();
}
Expand All @@ -1970,22 +1970,22 @@ private boolean isInputTransform(TransformMetaDataCombi combi) {
/** Stops all transforms from running, and alerts any registered listeners. */
@Override
public void stopAll() {
if (isAlreadyStopped.get()) {
if (!isAlreadyStopped.compareAndSet(false, true)) {
return;
}

// A pipeline that never got as far as allocating its transforms can still be stopped: it just
// has nothing to stop. Bailing out here used to leave it without a terminal state.
if (transforms != null) {
transforms.forEach(combi -> stopTransform(combi, false));
// Do not hold the synchronized list monitor while calling transform code.
new ArrayList<>(transforms).forEach(combi -> stopTransform(combi, false));
}

// if it is stopped it is not paused, nor is it still preparing or initializing
setPaused(false);
setPreparing(false);
setInitializing(false);
setStopped(true);
isAlreadyStopped.set(true);
if (debugController != null) {
debugController.cancelWaiters();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3246,33 +3246,26 @@ public void setInternalVariables() {
* @see org.apache.hop.pipeline.transform.ITransform#markStop()
*/
@Override
public synchronized void markStop() {

// Only mark a transform as stopped once
//
if (!markStopped.get()) {
markStopped.set(true);

Calendar cal = Calendar.getInstance();
stopTime = cal.getTime();

// We're finally completely done with this transform.
//
public void markStop() {
synchronized (this) {
// Publish the terminal state once, but never call external listeners under this monitor.
// A listener can stop the pipeline and acquire another transform's monitor.
if (!markStopped.compareAndSet(false, true)) {
return;
}
stopTime = Calendar.getInstance().getTime();
setRunning(false);

// Here we are completely done with the pipeline.
// Call all the attached listeners and notify the outside world that the transform has
// finished.
//
fireTransformFinishedListeners();
}
fireTransformFinishedListeners();
}

private synchronized void fireTransformFinishedListeners() {
private void fireTransformFinishedListeners() {
final List<ITransformFinishedListener> listeners;
synchronized (transformFinishedListeners) {
for (ITransformFinishedListener transformListener : transformFinishedListeners) {
transformListener.transformFinished(pipeline, transformMeta, this);
}
listeners = new ArrayList<>(transformFinishedListeners);
}
for (ITransformFinishedListener transformListener : listeners) {
transformListener.transformFinished(pipeline, transformMeta, this);
}
}

Expand Down
161 changes: 161 additions & 0 deletions engine/src/test/java/org/apache/hop/pipeline/PipelineTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,17 @@

package org.apache.hop.pipeline;

import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
Expand All @@ -33,7 +36,9 @@
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hop.core.Const;
import org.apache.hop.core.HopEnvironment;
Expand All @@ -48,11 +53,13 @@
import org.apache.hop.core.plugins.IPlugin;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
import org.apache.hop.pipeline.engine.EngineComponent.ComponentExecutionStatus;
import org.apache.hop.pipeline.engine.IPipelineEngine;
import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
import org.apache.hop.pipeline.transform.ITransform;
import org.apache.hop.pipeline.transform.ITransformData;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transform.TransformMetaDataCombi;
import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
Expand Down Expand Up @@ -631,4 +638,158 @@ void stopAllOnlyFiresTheStoppedListenersOnce() {

assertEquals(1, stopped[0]);
}

@Test
void stopAllCallbacksCanAccessTransformsFromAnotherThread() throws Exception {
assertStopCallbackCanAccessTransforms(false);
}

@Test
void safeStopCallbacksCanAccessTransformsFromAnotherThread() throws Exception {
assertStopCallbackCanAccessTransforms(true);
}

private void assertStopCallbackCanAccessTransforms(final boolean safeStop) throws Exception {
final var pipeline = new LocalPipelineEngine(new PipelineMeta());
pipeline.setLogChannel(mock(ILogChannel.class));
final var combi = new TransformMetaDataCombi();
combi.transformMeta = new TransformMeta("input", new DummyMeta());
combi.transform = mock(ITransform.class);
combi.data = mock(ITransformData.class);
pipeline.getPipelineMeta().addTransform(combi.transformMeta);
pipeline.setTransforms(Collections.synchronizedList(new ArrayList<>(List.of(combi))));
final var reader = Executors.newSingleThreadExecutor();
try {
doAnswer(
invocation -> {
// size() acquires the synchronized list's monitor. The callback must not hold it
// while waiting for another thread to inspect the pipeline.
final var size = reader.submit(() -> pipeline.getTransforms().size());
assertEquals(1, assertDoesNotThrow(() -> size.get(5, TimeUnit.SECONDS)));
return null;
})
.when(combi.transform)
.stopRunning();

if (safeStop) {
pipeline.safeStop();
} else {
pipeline.stopAll();
}

verify(combi.transform).stopRunning();
} finally {
reader.shutdownNow();
assertTrue(reader.awaitTermination(5, TimeUnit.SECONDS), "Transform reader did not finish");
}
}

@Test
void stopAllDoesNotReenterTransformStopCallbacks() throws Exception {
final var pipeline = new LocalPipelineEngine(new PipelineMeta());
pipeline.setLogChannel(mock(ILogChannel.class));
final var combi = new TransformMetaDataCombi();
combi.transform = mock(ITransform.class);
combi.data = mock(ITransformData.class);
pipeline.setTransforms(Collections.synchronizedList(new ArrayList<>(List.of(combi))));
final var callbacks = new AtomicInteger();
final var notifications = new AtomicInteger();
pipeline.addExecutionStoppedListener(p -> notifications.incrementAndGet());
doAnswer(
invocation -> {
// Bound a broken implementation to one recursive call instead of overflowing the
// stack: the observable callback count must still be exactly one.
if (callbacks.incrementAndGet() == 1) {
pipeline.stopAll();
}
return null;
})
.when(combi.transform)
.stopRunning();

pipeline.stopAll();

assertEquals(1, callbacks.get());
assertEquals(1, notifications.get());
assertTrue(pipeline.isStopped());
}

@Test
void concurrentStopAllCallsNotifyStoppedListenersOnce() throws Exception {
final var pipeline = new LocalPipelineEngine(new PipelineMeta());
pipeline.setLogChannel(mock(ILogChannel.class));
final var combi = new TransformMetaDataCombi();
combi.transform = mock(ITransform.class);
combi.data = mock(ITransformData.class);
// Isolate the one-shot guard from the synchronized list monitor: both calls can reach the
// callback in a broken implementation, without serializing on list iteration.
pipeline.setTransforms(new ArrayList<>(List.of(combi)));
final var callbackEntered = new CountDownLatch(1);
final var releaseCallback = new CountDownLatch(1);
final var callbacks = new AtomicInteger();
final var notifications = new AtomicInteger();
pipeline.addExecutionStoppedListener(p -> notifications.incrementAndGet());
doAnswer(
invocation -> {
if (callbacks.incrementAndGet() == 1) {
callbackEntered.countDown();
assertTrue(releaseCallback.await(10, TimeUnit.SECONDS), "Callback not released");
}
return null;
})
.when(combi.transform)
.stopRunning();
final var callers = Executors.newFixedThreadPool(2);
try {
final var first = callers.submit(pipeline::stopAll);
assertTrue(callbackEntered.await(5, TimeUnit.SECONDS), "First stop did not reach callback");
final var second = callers.submit(pipeline::stopAll);
// Keep the first stop inside the callback until the competing stop has returned.
second.get(5, TimeUnit.SECONDS);
releaseCallback.countDown();
first.get(5, TimeUnit.SECONDS);

assertEquals(1, notifications.get());
assertEquals(1, callbacks.get());
assertTrue(pipeline.isStopped());
} finally {
releaseCallback.countDown();
callers.shutdownNow();
assertTrue(callers.awaitTermination(5, TimeUnit.SECONDS), "Stop callers did not finish");
}
}

@Test
void safeStopOnlyStopsInputTransforms() throws Exception {
final var meta = new PipelineMeta();
final var input = new TransformMetaDataCombi();
input.transformMeta = new TransformMeta("input", new DummyMeta());
input.transform = mock(ITransform.class);
input.data = mock(ITransformData.class);
final var downstream = new TransformMetaDataCombi();
downstream.transformMeta = new TransformMeta("downstream", new DummyMeta());
downstream.transform = mock(ITransform.class);
downstream.data = mock(ITransformData.class);
meta.addTransform(input.transformMeta);
meta.addTransform(downstream.transformMeta);
meta.addPipelineHop(new PipelineHopMeta(input.transformMeta, downstream.transformMeta));
final var pipeline = new LocalPipelineEngine(meta);
pipeline.setLogChannel(mock(ILogChannel.class));
pipeline.setTransforms(
Collections.synchronizedList(new ArrayList<>(List.of(input, downstream))));

pipeline.safeStop();

verify(input.transform).setStopped(true);
verify(input.transform).setSafeStopped(true);
verify(input.transform).stopRunning();
verify(input.transform).setOutputDone();
verify(input.data).setStatus(ComponentExecutionStatus.STATUS_STOPPED);
verify(downstream.transform, never()).setStopped(true);
verify(downstream.transform, never()).setSafeStopped(true);
verify(downstream.transform, never()).stopRunning();
verify(downstream.transform, never()).setOutputDone();
verify(downstream.data, never()).setStatus(ComponentExecutionStatus.STATUS_STOPPED);
assertFalse(pipeline.isStopped(), "Downstream transforms must be allowed to drain");
}
}
Loading
Loading