diff --git a/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/ConcurrentClusterFlowChecker.java b/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/ConcurrentClusterFlowChecker.java index 9083bf929d..329f5b53c7 100644 --- a/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/ConcurrentClusterFlowChecker.java +++ b/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/ConcurrentClusterFlowChecker.java @@ -95,6 +95,12 @@ public static TokenResult releaseConcurrentToken(/*@Valid*/ long tokenId) { } int acquireCount = node.getAcquireCount(); AtomicInteger nowCalls = CurrentConcurrencyManager.get(node.getFlowId()); + if (nowCalls == null) { + // The concurrency counter has been removed by a concurrent rule refresh, + // there is nothing left to release. + RecordLog.info("[ConcurrentClusterFlowChecker] The concurrency counter of flowId<{}> is absent, skip releasing", node.getFlowId()); + return new TokenResult(TokenResultStatus.RELEASE_OK); + } nowCalls.getAndAdd(-1 * acquireCount); ClusterServerStatLogUtil.log("concurrent|release|" + rule.getClusterConfig().getFlowId(), acquireCount); return new TokenResult(TokenResultStatus.RELEASE_OK); diff --git a/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategy.java b/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategy.java index 24fd42fbfc..349e4e8ac7 100644 --- a/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategy.java +++ b/sentinel-cluster/sentinel-cluster-server-default/src/main/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategy.java @@ -21,6 +21,7 @@ import com.alibaba.csp.sentinel.cluster.server.connection.ConnectionManager; import com.alibaba.csp.sentinel.concurrent.NamedThreadFactory; import com.alibaba.csp.sentinel.log.RecordLog; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; import com.alibaba.csp.sentinel.util.AssertUtil; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; @@ -115,7 +116,16 @@ private void clearToken() { // If we find that token's save time is more than 2 times of the client's call resource timeout time, // the token will be determined to timeout. - long resourceTimeout = ClusterFlowRuleManager.getFlowRuleById(node.getFlowId()).getClusterConfig().getResourceTimeout(); + FlowRule rule = ClusterFlowRuleManager.getFlowRuleById(node.getFlowId()); + if (rule == null) { + // The rule has been removed by a hot update, so the token will never be + // released by its client. Recycle it directly, otherwise the thrown NPE + // would abort this round of scanning and starve all following expired tokens. + removeToken(key, node); + RecordLog.info("[RegularExpireStrategy] Delete the token<{}> whose rule is absent for flowId<{}>", node.getTokenId(), node.getFlowId()); + continue; + } + long resourceTimeout = rule.getClusterConfig().getResourceTimeout(); if (System.currentTimeMillis() - node.getResourceTimeout() > resourceTimeout) { removeToken(key, node); RecordLog.info("[RegularExpireStrategy] Delete the expired token<{}> because of resource timeout for ruleId<{}>", node.getTokenId(), node.getFlowId()); diff --git a/sentinel-cluster/sentinel-cluster-server-default/src/test/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategyNpeTest.java b/sentinel-cluster/sentinel-cluster-server-default/src/test/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategyNpeTest.java new file mode 100644 index 0000000000..09a6795020 --- /dev/null +++ b/sentinel-cluster/sentinel-cluster-server-default/src/test/java/com/alibaba/csp/sentinel/cluster/flow/statistic/concurrent/expire/RegularExpireStrategyNpeTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.csp.sentinel.cluster.flow.statistic.concurrent.expire; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + +import com.alibaba.csp.sentinel.cluster.TokenResult; +import com.alibaba.csp.sentinel.cluster.TokenResultStatus; +import com.alibaba.csp.sentinel.cluster.flow.ConcurrentClusterFlowChecker; +import com.alibaba.csp.sentinel.cluster.flow.rule.ClusterFlowRuleManager; +import com.alibaba.csp.sentinel.cluster.flow.statistic.concurrent.CurrentConcurrencyManager; +import com.alibaba.csp.sentinel.cluster.flow.statistic.concurrent.TokenCacheNode; +import com.alibaba.csp.sentinel.cluster.flow.statistic.concurrent.TokenCacheNodeManager; +import com.alibaba.csp.sentinel.slots.block.ClusterRuleConstant; +import com.alibaba.csp.sentinel.slots.block.RuleConstant; +import com.alibaba.csp.sentinel.slots.block.flow.ClusterFlowConfig; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +/** + * Regression tests for the expired-token cleanup path. + * + *

{@code RegularExpireStrategy#clearToken} used to call + * {@code ClusterFlowRuleManager.getFlowRuleById(flowId).getClusterConfig()} without a null + * check. After a rule is removed by a hot update, {@code getFlowRuleById} returns null and + * the cleanup task throws an NPE. Worse, the for-loop always scans from the head of the + * key list, the aborted element is never removed, so every following round aborts on the + * same node — all expired tokens behind it starve and their concurrency counters leak. + * + *

Same-origin race in {@code ConcurrentClusterFlowChecker#releaseConcurrentToken}: the + * concurrency counter can already be removed by a concurrent rule refresh, so + * {@code CurrentConcurrencyManager.get(flowId)} may return null. + */ +public class RegularExpireStrategyNpeTest { + + private static final String NAMESPACE = "test-regular-expire-namespace"; + + @After + public void tearDown() { + ClusterFlowRuleManager.loadRules(NAMESPACE, new ArrayList()); + CurrentConcurrencyManager.remove(888L); + CurrentConcurrencyManager.remove(999L); + } + + @Test + public void clearTokenShouldNotThrowWhenRuleHasBeenDeleted() throws Exception { + // The flowId is never registered, which simulates a rule removed by a hot update. + TokenCacheNode node = buildNode(12345L, 1, 10_000L, 10_000L); + + ConcurrentLinkedHashMap cache = buildCache(node); + + invokeClearToken(cache); + + // The token should be recycled instead of aborting the scan with an NPE. + Assert.assertNull("token of a deleted rule should be recycled by the cleanup task", + cache.get(node.getTokenId())); + } + + @Test + public void clearTokenShouldContinueScanningBehindNodeWithAbsentRule() throws Exception { + // Node B's rule is registered normally; its token has already timed out + // (resourceTimeout lies in the past while resourceTimeoutMs is 1000). + FlowRule ruleB = buildRule(888L, 10_000L, 1_000L); + registerRule(ruleB); + TokenCacheNode nodeA = buildNode(12345L, 1, 10_000L, 10_000L); + TokenCacheNode nodeB = buildNode(888L, 1, 10_000L, -5_000L); + // applyClusterFlowRule seeds the counter with 0; set a non-zero value to verify release. + CurrentConcurrencyManager.put(888L, 5); + + ConcurrentLinkedHashMap cache = buildCache(nodeA, nodeB); + + invokeClearToken(cache); + + // Without the fix the scan aborts on node A with an NPE and node B starves forever. + Assert.assertNull("node with absent rule should be recycled", cache.get(nodeA.getTokenId())); + Assert.assertNull("expired node behind an absent-rule node should also be recycled", + cache.get(nodeB.getTokenId())); + Assert.assertEquals("concurrency counter of node B should be released by 1", + 4, CurrentConcurrencyManager.get(888L).get()); + } + + @Test + public void releaseConcurrentTokenShouldNotThrowWhenCounterHasBeenRemoved() throws Exception { + FlowRule rule = buildRule(999L, 10_000L, 10_000L); + registerRule(rule); + + TokenCacheNode node = buildNode(999L, 1, 10_000L, 10_000L); + TokenCacheNodeManager.putTokenCacheNode(node.getTokenId(), node); + + // Simulate the race window of a concurrent rule refresh, which removes the + // concurrency counter of flowId 999 before the release path reads it. + CurrentConcurrencyManager.remove(999L); + + // The release path should not throw an NPE when the counter is absent. + TokenResult result = ConcurrentClusterFlowChecker.releaseConcurrentToken(node.getTokenId()); + Assert.assertNotNull(result); + Assert.assertEquals(TokenResultStatus.RELEASE_OK, result.getStatus().intValue()); + Assert.assertNull("token should be removed from the cache", + TokenCacheNodeManager.getTokenCacheNode(node.getTokenId())); + } + + /** + * Builds a token node with the given flowId. {@code clientTimeoutOffsetMs} and + * {@code resourceTimeoutOffsetMs} are relative to now (negative means in the past). + */ + private TokenCacheNode buildNode(long flowId, int acquireCount, + long clientTimeoutOffsetMs, long resourceTimeoutOffsetMs) { + FlowRule rule = buildRule(flowId, clientTimeoutOffsetMs, 1_000L); + TokenCacheNode node = TokenCacheNode.generateTokenCacheNode(rule, acquireCount, "test-client"); + node.setTokenId(flowId * 1000 + acquireCount); + node.setClientTimeout(clientTimeoutOffsetMs); + node.setResourceTimeout(resourceTimeoutOffsetMs); + return node; + } + + private FlowRule buildRule(long flowId, long clientOfflineTimeMs, long resourceTimeoutMs) { + FlowRule rule = new FlowRule("testResource"); + ClusterFlowConfig config = new ClusterFlowConfig(); + config.setFlowId(flowId); + config.setClientOfflineTime(clientOfflineTimeMs); + config.setResourceTimeout(resourceTimeoutMs); + config.setThresholdType(ClusterRuleConstant.FLOW_THRESHOLD_GLOBAL); + rule.setClusterConfig(config); + // applyClusterFlowRule silently skips rules that are not in cluster mode. + rule.setClusterMode(true); + rule.setGrade(RuleConstant.FLOW_GRADE_THREAD); + rule.setCount(10); + return rule; + } + + private void registerRule(FlowRule rule) { + List rules = new ArrayList(); + rules.add(rule); + ClusterFlowRuleManager.registerPropertyIfAbsent(NAMESPACE); + ClusterFlowRuleManager.loadRules(NAMESPACE, rules); + } + + private ConcurrentLinkedHashMap buildCache(TokenCacheNode... nodes) { + ConcurrentLinkedHashMap cache = + new ConcurrentLinkedHashMap.Builder() + .maximumWeightedCapacity(1000L) + .build(); + for (TokenCacheNode node : nodes) { + cache.put(node.getTokenId(), node); + } + return cache; + } + + private void invokeClearToken(ConcurrentLinkedHashMap cache) throws Exception { + RegularExpireStrategy strategy = new RegularExpireStrategy(cache); + Method clearToken = RegularExpireStrategy.class.getDeclaredMethod("clearToken"); + clearToken.setAccessible(true); + try { + clearToken.invoke(strategy); + } catch (Exception e) { + Assert.fail("clearToken should not throw when the rule has been deleted" + + " (otherwise the scan aborts on the same node every round and the" + + " following expired tokens starve): " + e.getCause()); + } + } +}