From e1fbc4f08ec6b489ba38e8e7dffc62fa40066bed Mon Sep 17 00:00:00 2001 From: Mitch Gaffigan Date: Sat, 17 Oct 2026 15:54:48 -0500 Subject: [PATCH] Set TCP keepalive idle time so idle connections survive NAT TCP connections with keepalive enabled used the OS idle time before the first probe, typically 2 hours. That is far longer than the idle timeouts of common NAT devices and load balancers (4 minutes for an Azure load balancer, 350 seconds for an AWS NAT gateway), which drop idle connections silently. The next read on such a connection fails with errors like the "Connection timed out" reported in #495. TCP Listener and TCP Sender sockets can now get a shorter idle time wherever they already enable keepalive: when "Keep Connection Open" is on, and always for TCP Sender server mode. The new oie.tcp.keepalive.idle system property sets the idle time in seconds and defaults to 120. The "Keep Connection Open" tooltips on both connectors now mention it. When the TCP plugin starts with the server, it reads the OS idle time (net.ipv4.tcp_keepalive_time on Linux) from a new socket and decides once how keepalive sockets are handled: - OS value higher than the property: each keepalive socket gets TCP_KEEPIDLE set to the property value. - OS value equal or lower: sockets are left alone and keep the OS value, so an admin who already lowered it keeps their setting. - OS value can't be read: sockets get the property value. - Property 0 or less, or TCP_KEEPIDLE not supported (such as Windows on a JDK without JDK-8308593): sockets are left alone. The decision isn't revisited until the engine restarts. If the OS value changes while the engine is running, sockets that get the property value keep getting it, even if the OS value drops below it. Sockets that are left alone use the OS value, so they follow the change, including a raise above the property value. This is a system property rather than a mirth.properties setting because the TCP connector is a plugin. No plugin has its own key in mirth.properties, and ConfigurationController has no generic accessor for its values, so reading one would mean adding TCP-specific code to core. WebServiceDispatcher reads http.maxRedirects the same way. An invalid property value logs an error and falls back to 120. If the OS rejects the value when it's set on a socket, the first failure logs an error and later ones log at DEBUG. Both are errors rather than warnings because the default log4j configuration only shows ERROR for this logger, and either one means the configured idle time isn't in effect. The decision is logged once at INFO, with the OS value when it can be read and what a later OS change does; the default log4j configuration doesn't show it. All of this is logged at server startup, before any TCP connection, because TcpServicePlugin.start() loads SocketUtil. Keepalive is explicitly disabled again when "Keep Connection Open" is off, as before. Behavior change: with Linux's default probe interval and count (75 seconds, 9 probes), a peer that disappears from an idle connection is now detected after about 13 minutes instead of about 2 hours 11 minutes. Issue: https://github.com/OpenIntegrationEngine/engine/issues/495 Co-authored-by: Tony Germano Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Mitch Gaffigan Signed-off-by: Tony Germano --- .../connect/connectors/tcp/TcpListener.java | 2 +- .../connect/connectors/tcp/TcpSender.java | 4 +- server/conf/custom.vmoptions | 8 ++ .../connect/connectors/tcp/SocketUtil.java | 80 +++++++++++++++++++ .../connect/connectors/tcp/TcpDispatcher.java | 4 +- .../connect/connectors/tcp/TcpReceiver.java | 2 +- .../connectors/tcp/TcpServicePlugin.java | 4 +- .../connectors/tcp/SocketUtilTest.java | 48 +++++++++++ 8 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 server/src/test/java/com/mirth/connect/connectors/tcp/SocketUtilTest.java diff --git a/client/src/main/java/com/mirth/connect/connectors/tcp/TcpListener.java b/client/src/main/java/com/mirth/connect/connectors/tcp/TcpListener.java index ec02f90fb7..0f7d0869a2 100644 --- a/client/src/main/java/com/mirth/connect/connectors/tcp/TcpListener.java +++ b/client/src/main/java/com/mirth/connect/connectors/tcp/TcpListener.java @@ -505,7 +505,7 @@ private void initToolTips() { receiveTimeoutField.setToolTipText("The amount of time, in milliseconds, to wait without receiving a message before closing a connection."); bufferSizeField.setToolTipText("Use larger values for larger messages, and smaller values
for smaller messages. Generally, the default value is fine."); - toolTipText = "Select No to close the listening socket after a received message has finished processing.
Otherwise the socket will remain open until the sending system closes it. In that case,
messages will only be processed if data is received and either the receive timeout is reached,
the client closes the socket, or an end of message byte sequence has been detected."; + toolTipText = "Select No to close the listening socket after a received message has finished processing.
Otherwise the socket will remain open until the sending system closes it. In that case,
messages will only be processed if data is received and either the receive timeout is reached,
the client closes the socket, or an end of message byte sequence has been detected.

With Yes, TCP keepalive probes start after the connection has been idle for 120 seconds,
or for the server's OS keepalive time if that is lower. The server's oie.tcp.keepalive.idle
system property changes the 120 seconds."; keepConnectionOpenYesRadio.setToolTipText(toolTipText); keepConnectionOpenNoRadio.setToolTipText(toolTipText); diff --git a/client/src/main/java/com/mirth/connect/connectors/tcp/TcpSender.java b/client/src/main/java/com/mirth/connect/connectors/tcp/TcpSender.java index 7712d19cf7..ea80e17473 100644 --- a/client/src/main/java/com/mirth/connect/connectors/tcp/TcpSender.java +++ b/client/src/main/java/com/mirth/connect/connectors/tcp/TcpSender.java @@ -440,7 +440,7 @@ public void actionPerformed(ActionEvent evt) { keepConnectionOpenYesRadio.setBorder(BorderFactory.createEmptyBorder(0, 0, 0, 0)); keepConnectionOpenGroup.add(keepConnectionOpenYesRadio); keepConnectionOpenYesRadio.setText("Yes"); - keepConnectionOpenYesRadio.setToolTipText("Select Yes to keep the connection to the host open across multiple messages.
Select No to immediately the close the connection to the host after sending each message."); + keepConnectionOpenYesRadio.setToolTipText("Select Yes to keep the connection to the host open across multiple messages.
Select No to immediately the close the connection to the host after sending each message.

With Yes, TCP keepalive probes start after the connection has been idle for 120 seconds,
or for the server's OS keepalive time if that is lower. The server's oie.tcp.keepalive.idle
system property changes the 120 seconds."); keepConnectionOpenYesRadio.setMargin(new Insets(0, 0, 0, 0)); keepConnectionOpenYesRadio.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { @@ -452,7 +452,7 @@ public void actionPerformed(ActionEvent evt) { keepConnectionOpenNoRadio.setBorder(BorderFactory.createEmptyBorder(0, 0, 0, 0)); keepConnectionOpenGroup.add(keepConnectionOpenNoRadio); keepConnectionOpenNoRadio.setText("No"); - keepConnectionOpenNoRadio.setToolTipText("Select Yes to keep the connection to the host open across multiple messages.
Select No to immediately the close the connection to the host after sending each message."); + keepConnectionOpenNoRadio.setToolTipText("Select Yes to keep the connection to the host open across multiple messages.
Select No to immediately the close the connection to the host after sending each message.

With Yes, TCP keepalive probes start after the connection has been idle for 120 seconds,
or for the server's OS keepalive time if that is lower. The server's oie.tcp.keepalive.idle
system property changes the 120 seconds."); keepConnectionOpenNoRadio.setMargin(new Insets(0, 0, 0, 0)); keepConnectionOpenNoRadio.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { diff --git a/server/conf/custom.vmoptions b/server/conf/custom.vmoptions index e838de0905..3a84d62473 100644 --- a/server/conf/custom.vmoptions +++ b/server/conf/custom.vmoptions @@ -22,6 +22,14 @@ # # Any lines which do not match the special directives above will be passed through directly as # arguments to the java binary. +# +# Engine options set with -D: +# +# -Doie.tcp.keepalive.idle= +# : seconds a TCP connection with keepalive enabled can sit idle before the first +# : keepalive probe is sent. Defaults to 120, which keeps idle connections alive +# : through NAT devices and load balancers. Only lowers the OS default; a lower OS +# : setting is kept. 0 or less always uses the OS default. diff --git a/server/src/main/java/com/mirth/connect/connectors/tcp/SocketUtil.java b/server/src/main/java/com/mirth/connect/connectors/tcp/SocketUtil.java index a9830e4d7f..5f31ec71d9 100644 --- a/server/src/main/java/com/mirth/connect/connectors/tcp/SocketUtil.java +++ b/server/src/main/java/com/mirth/connect/connectors/tcp/SocketUtil.java @@ -13,14 +13,28 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; +import java.net.SocketException; import java.net.UnknownHostException; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.mirth.connect.util.TcpUtil; +import jdk.net.ExtendedSocketOptions; + public class SocketUtil { + private static final Logger logger = LogManager.getLogger(SocketUtil.class); + + static final String KEEPALIVE_IDLE_PROPERTY = "oie.tcp.keepalive.idle"; + static final int DEFAULT_KEEPALIVE_IDLE = 120; + private static final Optional KEEPALIVE_IDLE = initKeepAliveIdle(); + private static final AtomicBoolean keepAliveIdleFailed = new AtomicBoolean(); + public static Socket createSocket(TcpConfiguration configuration) throws UnknownHostException, IOException { return createSocket(configuration, null); } @@ -67,6 +81,72 @@ public static void connectSocket(Socket socket, String host, int port, int timeo socket.connect(new InetSocketAddress(InetAddress.getByName(TcpUtil.getFixedHost(host)), port), timeout); } + /** + * Loads this class, which decides and logs the keepalive idle time. Called when the TCP plugin + * starts so the decision, and any error in the oie.tcp.keepalive.idle system property, is + * logged at server startup instead of at the first TCP connection. + */ + public static void init() {} + + /** + * Enables or disables TCP keepalive on a socket. When enabling, also lowers the idle time before + * the first keepalive probe to the oie.tcp.keepalive.idle system property (in seconds) if the + * OS default was higher when this class was loaded and the socket supports it. A value of 0 or + * less leaves the OS default. + */ + public static void setKeepAlive(Socket socket, boolean keepAlive) throws SocketException { + socket.setKeepAlive(keepAlive); + + if (keepAlive && KEEPALIVE_IDLE.isPresent() && socket.supportedOptions().contains(ExtendedSocketOptions.TCP_KEEPIDLE)) { + Integer idle = KEEPALIVE_IDLE.get(); + try { + socket.setOption(ExtendedSocketOptions.TCP_KEEPIDLE, idle); + } catch (IOException | IllegalArgumentException e) { + if (keepAliveIdleFailed.compareAndSet(false, true)) { + logger.error("Failed to set TCP_KEEPIDLE to {} seconds; using the OS default. Further failures will be logged at DEBUG.", idle, e); + } else { + logger.debug("Failed to set TCP_KEEPIDLE to {} seconds.", idle, e); + } + } + } + } + + private static Optional initKeepAliveIdle() { + int idle = DEFAULT_KEEPALIVE_IDLE; + String value = System.getProperty(KEEPALIVE_IDLE_PROPERTY); + if (value != null) { + try { + idle = Integer.parseInt(value.trim()); + } catch (NumberFormatException e) { + logger.error("Invalid value '{}' for {}; using {} seconds.", value, KEEPALIVE_IDLE_PROPERTY, DEFAULT_KEEPALIVE_IDLE); + } + } + + if (idle <= 0) { + logger.info("TCP connectors use the OS keepalive idle time; {} is {}.", KEEPALIVE_IDLE_PROPERTY, idle); + return Optional.empty(); + } + + try (Socket socket = new Socket()) { + if (!socket.supportedOptions().contains(ExtendedSocketOptions.TCP_KEEPIDLE)) { + logger.info("TCP connectors use the OS keepalive idle time; TCP_KEEPIDLE is not supported."); + return Optional.empty(); + } + + int osIdle = socket.getOption(ExtendedSocketOptions.TCP_KEEPIDLE); + if (osIdle <= idle) { + logger.info("TCP connectors use the OS keepalive idle time; its current value of {} seconds is not higher than {} ({} seconds). Later changes to the OS value still apply, but {} is not compared again until the server restarts.", osIdle, KEEPALIVE_IDLE_PROPERTY, idle, KEEPALIVE_IDLE_PROPERTY); + return Optional.empty(); + } + + logger.info("TCP connectors use a keepalive idle time of {} seconds instead of the current OS value of {} seconds. Later changes to the OS value are ignored until the server restarts.", idle, osIdle); + } catch (IOException e) { + logger.info("TCP connectors use a keepalive idle time of {} seconds; the current OS value could not be read. Later changes to the OS value are ignored until the server restarts.", idle); + } + + return Optional.of(idle); + } + public static void closeSocket(Socket socket) throws IOException { if (socket != null) { /* diff --git a/server/src/main/java/com/mirth/connect/connectors/tcp/TcpDispatcher.java b/server/src/main/java/com/mirth/connect/connectors/tcp/TcpDispatcher.java index 16dd25a411..192e25a206 100644 --- a/server/src/main/java/com/mirth/connect/connectors/tcp/TcpDispatcher.java +++ b/server/src/main/java/com/mirth/connect/connectors/tcp/TcpDispatcher.java @@ -376,7 +376,7 @@ public Response send(ConnectorProperties connectorProperties, ConnectorMessage m socket.setReceiveBufferSize(bufferSize); socket.setSendBufferSize(bufferSize); socket.setSoTimeout(responseTimeout); - socket.setKeepAlive(tcpDispatcherProperties.isKeepConnectionOpen()); + SocketUtil.setKeepAlive(socket, tcpDispatcherProperties.isKeepConnectionOpen()); eventController.dispatchEvent(new ConnectorCountEvent(getChannelId(), getMetaDataId(), getDestinationName(), ConnectionStatusEventType.CONNECTED, SocketUtil.getLocalAddress(socket) + " -> " + SocketUtil.getInetAddress(socket), true)); } @@ -810,7 +810,7 @@ private void initSocket(Socket socket) throws SocketException { socket.setReceiveBufferSize(bufferSize); socket.setSendBufferSize(bufferSize); socket.setSoTimeout(responseTimeout); - socket.setKeepAlive(true); + SocketUtil.setKeepAlive(socket, true); socket.setReuseAddress(true); socket.setTcpNoDelay(true); } diff --git a/server/src/main/java/com/mirth/connect/connectors/tcp/TcpReceiver.java b/server/src/main/java/com/mirth/connect/connectors/tcp/TcpReceiver.java index 66f48433db..3c36d18bae 100644 --- a/server/src/main/java/com/mirth/connect/connectors/tcp/TcpReceiver.java +++ b/server/src/main/java/com/mirth/connect/connectors/tcp/TcpReceiver.java @@ -1067,7 +1067,7 @@ private void initSocket(Socket socket) throws SocketException { socket.setReceiveBufferSize(bufferSize); socket.setSendBufferSize(bufferSize); socket.setSoTimeout(timeout); - socket.setKeepAlive(connectorProperties.isKeepConnectionOpen()); + SocketUtil.setKeepAlive(socket, connectorProperties.isKeepConnectionOpen()); socket.setReuseAddress(true); socket.setTcpNoDelay(true); } diff --git a/server/src/main/java/com/mirth/connect/connectors/tcp/TcpServicePlugin.java b/server/src/main/java/com/mirth/connect/connectors/tcp/TcpServicePlugin.java index d7ded156cd..e8c3f81aa2 100644 --- a/server/src/main/java/com/mirth/connect/connectors/tcp/TcpServicePlugin.java +++ b/server/src/main/java/com/mirth/connect/connectors/tcp/TcpServicePlugin.java @@ -25,7 +25,9 @@ public String getPluginPointName() { } @Override - public void start() {} + public void start() { + SocketUtil.init(); + } @Override public void stop() {} diff --git a/server/src/test/java/com/mirth/connect/connectors/tcp/SocketUtilTest.java b/server/src/test/java/com/mirth/connect/connectors/tcp/SocketUtilTest.java new file mode 100644 index 0000000000..eebfb5e099 --- /dev/null +++ b/server/src/test/java/com/mirth/connect/connectors/tcp/SocketUtilTest.java @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MPL-2.0 +// SPDX-FileCopyrightText: 2026 Tony Germano +package com.mirth.connect.connectors.tcp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.net.Socket; + +import org.junit.Assume; +import org.junit.Test; + +import jdk.net.ExtendedSocketOptions; + +public class SocketUtilTest { + + @Test + public void setKeepAliveEnablesKeepAliveAndCapsIdle() throws Exception { + assertNull("Test expects " + SocketUtil.KEEPALIVE_IDLE_PROPERTY + " to be unset", System.getProperty(SocketUtil.KEEPALIVE_IDLE_PROPERTY)); + + try (Socket socket = new Socket()) { + Assume.assumeTrue(socket.supportedOptions().contains(ExtendedSocketOptions.TCP_KEEPIDLE)); + int osIdle = socket.getOption(ExtendedSocketOptions.TCP_KEEPIDLE); + + SocketUtil.setKeepAlive(socket, true); + + assertTrue(socket.getKeepAlive()); + assertEquals(Math.min(osIdle, SocketUtil.DEFAULT_KEEPALIVE_IDLE), (int) socket.getOption(ExtendedSocketOptions.TCP_KEEPIDLE)); + } + } + + @Test + public void setKeepAliveDisablesKeepAliveAndLeavesIdle() throws Exception { + try (Socket socket = new Socket()) { + socket.setKeepAlive(true); + Integer idle = socket.supportedOptions().contains(ExtendedSocketOptions.TCP_KEEPIDLE) ? socket.getOption(ExtendedSocketOptions.TCP_KEEPIDLE) : null; + + SocketUtil.setKeepAlive(socket, false); + + assertFalse(socket.getKeepAlive()); + if (idle != null) { + assertEquals(idle, socket.getOption(ExtendedSocketOptions.TCP_KEEPIDLE)); + } + } + } +}