From 0995a656f54aa0a09aacf39451822115929f1636 Mon Sep 17 00:00:00 2001 From: konlue <2362586639@qq.com> Date: Sun, 11 Oct 2026 14:24:49 +0800 Subject: [PATCH] fix: prevent NPE and scan starvation in expired token cleanup when the flow rule is removed - RegularExpireStrategy#clearToken: null-check the rule looked up by flowId and recycle the token directly when the rule has been removed by a hot update, so the scan keeps going instead of aborting on the same node every round (which starves all following expired tokens and leaks the concurrency counters) - ConcurrentClusterFlowChecker#releaseConcurrentToken: guard against the concurrency counter already being removed by a concurrent rule refresh --- .../flow/ConcurrentClusterFlowChecker.java | 6 + .../expire/RegularExpireStrategy.java | 12 +- .../expire/RegularExpireStrategyNpeTest.java | 179 ++++++++++++++++++ 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 sentinel-cluster/sentinel-cluster-server-default/src/test/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategyNpeTest.java diff --git a/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/ConcurrentClusterFlowChecker.java b/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/ConcurrentClusterFlowChecker.java index 9083bf929d..329f5b53c7 100644 --- a/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/ConcurrentClusterFlowChecker.java +++ b/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/ConcurrentClusterFlowChecker.java @@ -95,6 +95,12 @@ public static TokenResult releaseConcurrentToken(/*@Valid*/ long tokenId) { } int acquireCount = node.getAcquireCount(); AtomicInteger nowCalls = CurrentConcurrencyManager.get(node.getFlowId()); + if (nowCalls == null) { + // The concurrency counter has been removed by a concurrent rule refresh, + // there is nothing left to release. + RecordLog.info("[ConcurrentClusterFlowChecker] The concurrency counter of flowId<{}> is absent, skip releasing", node.getFlowId()); + return new TokenResult(TokenResultStatus.RELEASE_OK); + } nowCalls.getAndAdd(-1 * acquireCount); ClusterServerStatLogUtil.log("concurrent|release|" + rule.getClusterConfig().getFlowId(), acquireCount); return new TokenResult(TokenResultStatus.RELEASE_OK); diff --git a/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategy.java b/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategy.java index 24fd42fbfc..349e4e8ac7 100644 --- a/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategy.java +++ b/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategy.java @@ -21,6 +21,7 @@ import com.alibaba.csp.sentinel.cluster.server.connection.ConnectionManager; import com.alibaba.csp.sentinel.concurrent.NamedThreadFactory; import com.alibaba.csp.sentinel.log.RecordLog; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; import com.alibaba.csp.sentinel.util.AssertUtil; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; @@ -115,7 +116,16 @@ private void clearToken() { // If we find that token's save time is more than 2 times of the client's call resource timeout time, // the token will be determined to timeout. - long resourceTimeout = ClusterFlowRuleManager.getFlowRuleById(node.getFlowId()).getClusterConfig().getResourceTimeout(); + FlowRule rule = ClusterFlowRuleManager.getFlowRuleById(node.getFlowId()); + if (rule == null) { + // The rule has been removed by a hot update, so the token will never be + // released by its client. Recycle it directly, otherwise the thrown NPE + // would abort this round of scanning and starve all following expired tokens. + removeToken(key, node); + RecordLog.info("[RegularExpireStrategy] Delete the token<{}> whose rule is absent for flowId<{}>", node.getTokenId(), node.getFlowId()); + continue; + } + long resourceTimeout = rule.getClusterConfig().getResourceTimeout(); if (System.currentTimeMillis() - node.getResourceTimeout() > resourceTimeout) { removeToken(key, node); RecordLog.info("[RegularExpireStrategy] Delete the expired token<{}> because of resource timeout for ruleId<{}>", node.getTokenId(), node.getFlowId()); diff --git a/sentinel-cluster/sentinel-cluster-server-default/src/test/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategyNpeTest.java b/sentinel-cluster/sentinel-cluster-server-default/src/test/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategyNpeTest.java new file mode 100644 index 0000000000..09a6795020 --- /dev/null +++ b/sentinel-cluster/sentinel-cluster-server-default/src/test/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategyNpeTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.csp.sentinel.cluster.flow.statistic.concurrent.expire; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + +import com.alibaba.csp.sentinel.cluster.TokenResult; +import com.alibaba.csp.sentinel.cluster.TokenResultStatus; +import com.alibaba.csp.sentinel.cluster.flow.ConcurrentClusterFlowChecker; +import com.alibaba.csp.sentinel.cluster.flow.rule.ClusterFlowRuleManager; +import com.alibaba.csp.sentinel.cluster.flow.statistic.concurrent.CurrentConcurrencyManager; +import com.alibaba.csp.sentinel.cluster.flow.statistic.concurrent.TokenCacheNode; +import com.alibaba.csp.sentinel.cluster.flow.statistic.concurrent.TokenCacheNodeManager; +import com.alibaba.csp.sentinel.slots.block.ClusterRuleConstant; +import com.alibaba.csp.sentinel.slots.block.RuleConstant; +import com.alibaba.csp.sentinel.slots.block.flow.ClusterFlowConfig; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +/** + * Regression tests for the expired-token cleanup path. + * + *
{@code RegularExpireStrategy#clearToken} used to call + * {@code ClusterFlowRuleManager.getFlowRuleById(flowId).getClusterConfig()} without a null + * check. After a rule is removed by a hot update, {@code getFlowRuleById} returns null and + * the cleanup task throws an NPE. Worse, the for-loop always scans from the head of the + * key list, the aborted element is never removed, so every following round aborts on the + * same node — all expired tokens behind it starve and their concurrency counters leak. + * + *
Same-origin race in {@code ConcurrentClusterFlowChecker#releaseConcurrentToken}: the
+ * concurrency counter can already be removed by a concurrent rule refresh, so
+ * {@code CurrentConcurrencyManager.get(flowId)} may return null.
+ */
+public class RegularExpireStrategyNpeTest {
+
+ private static final String NAMESPACE = "test-regular-expire-namespace";
+
+ @After
+ public void tearDown() {
+ ClusterFlowRuleManager.loadRules(NAMESPACE, new ArrayList