From 0c6097bb44156f94d2019f8790651204c90db25c Mon Sep 17 00:00:00 2001 From: shizhili Date: Fri, 8 Dec 2023 09:37:11 +0800 Subject: [PATCH] backport support kv storage for consumequeue --- ...-Support-KV-Storage-for-ConsumeQueue.patch | 29480 ++++++++++++++++ rocketmq.spec | 6 +- 2 files changed, 29485 insertions(+), 1 deletion(-) create mode 100644 patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch diff --git a/patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch b/patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch new file mode 100644 index 0000000..a4a09ae --- /dev/null +++ b/patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch @@ -0,0 +1,29480 @@ +From 0a046b85611bdc93c529650dc951f4cabbb61db9 Mon Sep 17 00:00:00 2001 +From: zhiliatom +Date: Fri, 8 Dec 2023 09:25:41 +0800 +Subject: [PATCH] [ISSUE #7064] [RIP-66-2] Support KV(RocksDB) Storage for + ConsumeQueue + +--- + .../rocketmq/broker/BrokerController.java | 74 +- + .../broker/controller/ReplicasManager.java | 4 +- + .../offset/RocksDBConsumerOffsetManager.java | 8 +- + .../broker/processor/AckMessageProcessor.java | 2 +- + .../processor/AdminBrokerProcessor.java | 28 +- + .../ChangeInvisibleTimeProcessor.java | 4 +- + .../processor/PopBufferMergeService.java | 4 +- + .../broker/processor/PopMessageProcessor.java | 2 +- + .../broker/processor/PopReviveService.java | 6 +- + .../RocksDBSubscriptionGroupManager.java | 10 +- + .../topic/RocksDBTopicConfigManager.java | 8 +- + .../processor/PopReviveServiceTest.java | 4 +- + common/pom.xml | 2 +- + .../org/apache/rocketmq/common/MixAll.java | 1 + + .../rocketmq/common/attribute/CQType.java | 3 +- + .../common/config/AbstractRocksDBStorage.java | 112 +- + .../common/config/ConfigRocksDBStorage.java | 2 +- + .../common/message/MessageExtBrokerInner.java | 13 + + .../rocketmq/common/topic/TopicValidator.java | 2 + + .../rocketmq/common/utils/DataConverter.java | 2 +- + .../rocketmq/example/quickstart/Producer.java | 2 +- + metrics | 241 ++ + patch009-modify-CRLF-to-LF.patch | 127 + + patch010-backport-add-some-fixes | 1286 ++++++ + patch011-backport-optimize-config | 1390 +++++++ + ...optimize-opentelemetry-metric-config.patch | 2081 ++++++++++ + ...-backport-enhance-rockdbconfigtojson.patch | 2920 +++++++++++++ + patch013-backport-enhance-admin-output.patch | 892 ++++ + ...ueue-Selection-Strategy-Optimization.patch | 2023 +++++++++ + patch015-backport-fix-some-bugs.patch | 1894 +++++++++ + ...rt-Optimize-fault-tolerant-mechanism.patch | 520 +++ + ...port-Convergent-thread-pool-creation.patch | 2243 ++++++++++ + ...ckport-enhancement-of-tiered-storage.patch | 601 +++ + patch019-backport-some-bugfix.patch | 1499 +++++++ + patch020-backport-add-goaway-mechanism.patch | 3696 +++++++++++++++++ + patch021-backport-some-enhancements.patch | 344 ++ + pom.xml | 4 +- + store/pom.xml | 4 + + .../org/apache/rocketmq/store/CommitLog.java | 94 +- + .../rocketmq/store/CommitLogDispatcher.java | 5 +- + .../apache/rocketmq/store/ConsumeQueue.java | 93 +- + .../rocketmq/store/DefaultMessageStore.java | 279 +- + .../apache/rocketmq/store/MessageStore.java | 54 +- + .../rocketmq/store/RocksDBMessageStore.java | 169 + + .../apache/rocketmq/store/RunningFlags.java | 22 +- + .../store/config/MessageStoreConfig.java | 33 +- + .../store/dledger/DLedgerCommitLog.java | 53 +- + .../apache/rocketmq/store/ha/HAService.java | 3 +- + .../ha/autoswitch/AutoSwitchHAClient.java | 2 +- + .../ha/autoswitch/AutoSwitchHAService.java | 7 +- + .../plugin/AbstractPluginMessageStore.java | 42 +- + .../queue/AbstractConsumeQueueStore.java | 105 + + .../store/queue/BatchConsumeQueue.java | 20 + + .../store/queue/ConsumeQueueInterface.java | 27 +- + .../store/queue/ConsumeQueueStore.java | 293 +- + .../queue/ConsumeQueueStoreInterface.java | 289 ++ + .../rocketmq/store/queue/MultiDispatch.java | 76 + + .../store/queue/QueueOffsetOperator.java | 8 + + .../store/queue/RocksDBConsumeQueue.java | 437 ++ + .../queue/RocksDBConsumeQueueOffsetTable.java | 641 +++ + .../store/queue/RocksDBConsumeQueueStore.java | 441 ++ + .../store/queue/RocksDBConsumeQueueTable.java | 312 ++ + .../ConsumeQueueCompactionFilterFactory.java | 47 + + .../rocksdb/ConsumeQueueRocksDBStorage.java | 133 + + .../store/rocksdb/RocksDBOptionsFactory.java | 161 + + .../store/timer/TimerMessageStore.java | 65 +- + .../store/DefaultMessageStoreTest.java | 5 +- + .../rocketmq/store/MultiDispatchTest.java | 8 +- + .../store/RocksDBMessageStoreTest.java | 1060 +++++ + .../apache/rocketmq/store/StoreTestUtil.java | 9 +- + .../rocketmq/store/ha/HAServerTest.java | 16 +- + .../store/ha/autoswitch/AutoSwitchHATest.java | 3 +- + .../tieredstore/TieredMessageStore.java | 15 +- + .../ExportMetadataInRocksDBCommand.java | 8 +- + .../metadata/RocksDBConfigToJsonCommand.java | 118 + + 75 files changed, 26543 insertions(+), 668 deletions(-) + create mode 100644 metrics + create mode 100644 patch009-modify-CRLF-to-LF.patch + create mode 100644 patch010-backport-add-some-fixes + create mode 100644 patch011-backport-optimize-config + create mode 100644 patch011-backport-optimize-opentelemetry-metric-config.patch + create mode 100644 patch012-backport-enhance-rockdbconfigtojson.patch + create mode 100644 patch013-backport-enhance-admin-output.patch + create mode 100644 patch014-backport-Queue-Selection-Strategy-Optimization.patch + create mode 100644 patch015-backport-fix-some-bugs.patch + create mode 100644 patch016-backport-Optimize-fault-tolerant-mechanism.patch + create mode 100644 patch017-backport-Convergent-thread-pool-creation.patch + create mode 100644 patch018-backport-enhancement-of-tiered-storage.patch + create mode 100644 patch019-backport-some-bugfix.patch + create mode 100644 patch020-backport-add-goaway-mechanism.patch + create mode 100644 patch021-backport-some-enhancements.patch + create mode 100644 store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java + create mode 100644 store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java + create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index d4bded600..9f1fd0ad0 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -16,7 +16,33 @@ + */ + package org.apache.rocketmq.broker; + ++import java.io.IOException; ++import java.net.InetSocketAddress; ++import java.util.AbstractMap; ++import java.util.ArrayList; ++import java.util.Arrays; ++import java.util.Collections; ++import java.util.HashMap; ++import java.util.List; ++import java.util.Map; ++import java.util.Objects; ++import java.util.Optional; ++import java.util.concurrent.BlockingQueue; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++import java.util.concurrent.CountDownLatch; ++import java.util.concurrent.ExecutorService; ++import java.util.concurrent.LinkedBlockingQueue; ++import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.ScheduledFuture; ++import java.util.concurrent.TimeUnit; ++import java.util.concurrent.locks.Lock; ++import java.util.concurrent.locks.ReentrantLock; ++import java.util.function.Function; ++import java.util.stream.Collectors; ++ + import com.google.common.collect.Lists; ++ + import org.apache.rocketmq.acl.AccessValidator; + import org.apache.rocketmq.acl.plain.PlainAccessValidator; + import org.apache.rocketmq.broker.client.ClientHousekeepingService; +@@ -126,7 +152,7 @@ import org.apache.rocketmq.store.DefaultMessageStore; + import org.apache.rocketmq.store.MessageArrivingListener; + import org.apache.rocketmq.store.MessageStore; + import org.apache.rocketmq.store.PutMessageResult; +-import org.apache.rocketmq.store.StoreType; ++import org.apache.rocketmq.store.RocksDBMessageStore; + import org.apache.rocketmq.store.config.BrokerRole; + import org.apache.rocketmq.store.config.MessageStoreConfig; + import org.apache.rocketmq.store.dledger.DLedgerCommitLog; +@@ -141,31 +167,6 @@ import org.apache.rocketmq.store.timer.TimerCheckpoint; + import org.apache.rocketmq.store.timer.TimerMessageStore; + import org.apache.rocketmq.store.timer.TimerMetrics; + +-import java.io.IOException; +-import java.net.InetSocketAddress; +-import java.util.AbstractMap; +-import java.util.ArrayList; +-import java.util.Arrays; +-import java.util.Collections; +-import java.util.HashMap; +-import java.util.List; +-import java.util.Map; +-import java.util.Objects; +-import java.util.Optional; +-import java.util.concurrent.BlockingQueue; +-import java.util.concurrent.ConcurrentHashMap; +-import java.util.concurrent.ConcurrentMap; +-import java.util.concurrent.CountDownLatch; +-import java.util.concurrent.ExecutorService; +-import java.util.concurrent.LinkedBlockingQueue; +-import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledFuture; +-import java.util.concurrent.TimeUnit; +-import java.util.concurrent.locks.Lock; +-import java.util.concurrent.locks.ReentrantLock; +-import java.util.function.Function; +-import java.util.stream.Collectors; +- + public class BrokerController { + protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); +@@ -308,7 +309,7 @@ public class BrokerController { + this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort())); + this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()); + this.broadcastOffsetManager = new BroadcastOffsetManager(this); +- if (isEnableRocksDBStore()) { ++ if (this.messageStoreConfig.isEnableRocksDBStore()) { + this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this); + this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this); + this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqConsumerOffsetManager(this) : new RocksDBConsumerOffsetManager(this); +@@ -747,7 +748,12 @@ public class BrokerController { + public boolean initializeMessageStore() { + boolean result = true; + try { +- DefaultMessageStore defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); ++ DefaultMessageStore defaultMessageStore; ++ if (this.messageStoreConfig.isEnableRocksDBStore()) { ++ defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); ++ } else { ++ defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); ++ } + + if (messageStoreConfig.isEnableDLegerCommitLog()) { + DLedgerRoleChangeHandler roleChangeHandler = +@@ -944,16 +950,16 @@ public class BrokerController { + this.transactionalMessageService = ServiceProvider.loadClass(TransactionalMessageService.class); + if (null == this.transactionalMessageService) { + this.transactionalMessageService = new TransactionalMessageServiceImpl( +- new TransactionalMessageBridge(this, this.getMessageStore())); ++ new TransactionalMessageBridge(this, this.getMessageStore())); + LOG.warn("Load default transaction message hook service: {}", +- TransactionalMessageServiceImpl.class.getSimpleName()); ++ TransactionalMessageServiceImpl.class.getSimpleName()); + } + this.transactionalMessageCheckListener = ServiceProvider.loadClass( +- AbstractTransactionalMessageCheckListener.class); ++ AbstractTransactionalMessageCheckListener.class); + if (null == this.transactionalMessageCheckListener) { + this.transactionalMessageCheckListener = new DefaultTransactionalMessageCheckListener(); + LOG.warn("Load default discard message hook service: {}", +- DefaultTransactionalMessageCheckListener.class.getSimpleName()); ++ DefaultTransactionalMessageCheckListener.class.getSimpleName()); + } + this.transactionalMessageCheckListener.setBrokerController(this); + this.transactionalMessageCheckService = new TransactionalMessageCheckService(this); +@@ -2412,8 +2418,4 @@ public class BrokerController { + public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) { + this.coldDataCgCtrService = coldDataCgCtrService; + } +- +- public boolean isEnableRocksDBStore() { +- return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.messageStoreConfig.getStoreType()); +- } + } +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +index a989e6e68..a1d711cb2 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +@@ -224,7 +224,7 @@ public class ReplicasManager { + + public synchronized void changeBrokerRole(final Long newMasterBrokerId, final String newMasterAddress, + final Integer newMasterEpoch, +- final Integer syncStateSetEpoch, final Set syncStateSet) { ++ final Integer syncStateSetEpoch, final Set syncStateSet) throws Exception { + if (newMasterBrokerId != null && newMasterEpoch > this.masterEpoch) { + if (newMasterBrokerId.equals(this.brokerControllerId)) { + changeToMaster(newMasterEpoch, syncStateSetEpoch, syncStateSet); +@@ -234,7 +234,7 @@ public class ReplicasManager { + } + } + +- public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set syncStateSet) { ++ public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set syncStateSet) throws Exception { + synchronized (this) { + if (newMasterEpoch > this.masterEpoch) { + LOGGER.info("Begin to change to master, brokerName:{}, replicas:{}, new Epoch:{}", this.brokerConfig.getBrokerName(), this.brokerAddress, newMasterEpoch); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +index 5695a3356..05b53b0bc 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +@@ -33,7 +33,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + + public RocksDBConsumerOffsetManager(BrokerController brokerController) { + super(brokerController); +- this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); ++ this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + } + + @Override +@@ -49,7 +49,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + @Override + protected void removeConsumerOffset(String topicAtGroup) { + try { +- byte[] keyBytes = topicAtGroup.getBytes(DataConverter.charset); ++ byte[] keyBytes = topicAtGroup.getBytes(DataConverter.CHARSET_UTF8); + this.rocksDBConfigManager.delete(keyBytes); + } catch (Exception e) { + LOG.error("kv remove consumerOffset Failed, {}", topicAtGroup); +@@ -58,7 +58,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + + @Override + protected void decode0(final byte[] key, final byte[] body) { +- String topicAtGroup = new String(key, DataConverter.charset); ++ String topicAtGroup = new String(key, DataConverter.CHARSET_UTF8); + RocksDBOffsetSerializeWrapper wrapper = JSON.parseObject(body, RocksDBOffsetSerializeWrapper.class); + + this.offsetTable.put(topicAtGroup, wrapper.getOffsetTable()); +@@ -93,7 +93,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + } + + private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupName, final ConcurrentMap offsetMap) throws Exception { +- byte[] keyBytes = topicGroupName.getBytes(DataConverter.charset); ++ byte[] keyBytes = topicGroupName.getBytes(DataConverter.CHARSET_UTF8); + RocksDBOffsetSerializeWrapper wrapper = new RocksDBOffsetSerializeWrapper(); + wrapper.setOffsetTable(offsetMap); + byte[] valueBytes = JSON.toJSONBytes(wrapper, SerializerFeature.BrowserCompatible); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +index 244b459d6..59a3e63b2 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +@@ -253,7 +253,7 @@ public class AckMessageProcessor implements NettyRequestProcessor { + + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(reviveTopic); +- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(rqId); + if (ackMsg instanceof BatchAckMsg) { + msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +index e77120e15..dd4ec960f 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +@@ -539,14 +539,18 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + + final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic); + // delete pop retry topics first +- for (String group : groups) { +- final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); +- if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { +- deleteTopicInBroker(popRetryTopic); ++ try { ++ for (String group : groups) { ++ final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); ++ if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { ++ deleteTopicInBroker(popRetryTopic); ++ } + } ++ // delete topic ++ deleteTopicInBroker(topic); ++ } catch (Throwable t) { ++ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } +- // delete topic +- deleteTopicInBroker(topic); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; +@@ -2081,7 +2085,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + public RemotingCommand cleanExpiredConsumeQueue() { + LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: start."); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); +- brokerController.getMessageStore().cleanExpiredConsumerQueue(); ++ try { ++ brokerController.getMessageStore().cleanExpiredConsumerQueue(); ++ } catch (Throwable t) { ++ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); ++ } + LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: end."); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); +@@ -2781,7 +2789,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + + final ReplicasManager replicasManager = this.brokerController.getReplicasManager(); + if (replicasManager != null) { +- replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet()); ++ try { ++ replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet()); ++ } catch (Exception e) { ++ throw new RemotingCommandException(e.getMessage()); ++ } + } + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +index 2ccdf07f6..bdfffff09 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +@@ -180,7 +180,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { + } + + msgInner.setTopic(reviveTopic); +- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(rqId); + msgInner.setTags(PopAckConstants.ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +@@ -216,7 +216,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { + ck.addDiff(0); + ck.setBrokerName(brokerName); + +- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(reviveQid); + msgInner.setTags(PopAckConstants.CK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java +index b7ba8ad4a..8a85dd8fe 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java +@@ -633,7 +633,7 @@ public class PopBufferMergeService extends ServiceThread { + ackMsg.setQueueId(point.getQueueId()); + ackMsg.setPopTime(point.getPopTime()); + msgInner.setTopic(popMessageProcessor.reviveTopic); +- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(pointWrapper.getReviveQueueId()); + msgInner.setTags(PopAckConstants.ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +@@ -673,7 +673,7 @@ public class PopBufferMergeService extends ServiceThread { + batchAckMsg.setQueueId(point.getQueueId()); + batchAckMsg.setPopTime(point.getPopTime()); + msgInner.setTopic(popMessageProcessor.reviveTopic); +- msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(pointWrapper.getReviveQueueId()); + msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +index 0d9bdf143..f5d07c5aa 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +@@ -685,7 +685,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + + msgInner.setTopic(reviveTopic); +- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(reviveQid); + msgInner.setTags(PopAckConstants.CK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +index d5174d3d1..4f80752e1 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +@@ -356,7 +356,7 @@ public class PopReviveService extends ServiceThread { + } + for (MessageExt messageExt : messageExts) { + if (PopAckConstants.CK_TAG.equals(messageExt.getTags())) { +- String raw = new String(messageExt.getBody(), DataConverter.charset); ++ String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("reviveQueueId={},find ck, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + } +@@ -371,7 +371,7 @@ public class PopReviveService extends ServiceThread { + firstRt = point.getReviveTime(); + } + } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) { +- String raw = new String(messageExt.getBody(), DataConverter.charset); ++ String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("reviveQueueId={},find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + } +@@ -395,7 +395,7 @@ public class PopReviveService extends ServiceThread { + } + } + } else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) { +- String raw = new String(messageExt.getBody(), DataConverter.charset); ++ String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + } +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +index 6503970af..e9a81a8d6 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +@@ -30,7 +30,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + + public RocksDBSubscriptionGroupManager(BrokerController brokerController) { + super(brokerController, false); +- this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); ++ this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + } + + @Override +@@ -53,7 +53,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig); + + try { +- byte[] keyBytes = groupName.getBytes(DataConverter.charset); ++ byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8); + byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible); + this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); + } catch (Exception e) { +@@ -68,7 +68,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.putIfAbsent(groupName, subscriptionGroupConfig); + if (oldConfig == null) { + try { +- byte[] keyBytes = groupName.getBytes(DataConverter.charset); ++ byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8); + byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible); + this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); + } catch (Exception e) { +@@ -82,7 +82,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) { + SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.remove(groupName); + try { +- this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.charset)); ++ this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.CHARSET_UTF8)); + } catch (Exception e) { + log.error("kv delete sub Failed, {}", subscriptionGroupConfig.toString()); + } +@@ -91,7 +91,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + + @Override + protected void decode0(byte[] key, byte[] body) { +- String groupName = new String(key, DataConverter.charset); ++ String groupName = new String(key, DataConverter.CHARSET_UTF8); + SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class); + + this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +index 7da0d7c8a..fddecf2d9 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +@@ -30,7 +30,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { + + public RocksDBTopicConfigManager(BrokerController brokerController) { + super(brokerController, false); +- this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); ++ this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + } + + @Override +@@ -49,7 +49,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { + + @Override + protected void decode0(byte[] key, byte[] body) { +- String topicName = new String(key, DataConverter.charset); ++ String topicName = new String(key, DataConverter.CHARSET_UTF8); + TopicConfig topicConfig = JSON.parseObject(body, TopicConfig.class); + + this.topicConfigTable.put(topicName, topicConfig); +@@ -66,7 +66,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { + String topicName = topicConfig.getTopicName(); + TopicConfig oldTopicConfig = this.topicConfigTable.put(topicName, topicConfig); + try { +- byte[] keyBytes = topicName.getBytes(DataConverter.charset); ++ byte[] keyBytes = topicName.getBytes(DataConverter.CHARSET_UTF8); + byte[] valueBytes = JSON.toJSONBytes(topicConfig, SerializerFeature.BrowserCompatible); + this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); + } catch (Exception e) { +@@ -79,7 +79,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { + protected TopicConfig removeTopicConfig(String topicName) { + TopicConfig topicConfig = this.topicConfigTable.remove(topicName); + try { +- this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.charset)); ++ this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.CHARSET_UTF8)); + } catch (Exception e) { + log.error("kv remove topic Failed, {}", topicConfig.toString()); + } +diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java +index 1c3a0cd45..78b76264f 100644 +--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java ++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java +@@ -234,7 +234,7 @@ public class PopReviveServiceTest { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + + msgInner.setTopic(REVIVE_TOPIC); +- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(REVIVE_QUEUE_ID); + msgInner.setTags(PopAckConstants.CK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +@@ -269,7 +269,7 @@ public class PopReviveServiceTest { + SocketAddress host, long deliverMs, String ackUniqueId) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(reviveTopic); +- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(reviveQid); + msgInner.setTags(PopAckConstants.ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +diff --git a/common/pom.xml b/common/pom.xml +index 6104c3ac6..a28ed228f 100644 +--- a/common/pom.xml ++++ b/common/pom.xml +@@ -109,7 +109,7 @@ + rocketmq-logback-classic + + +- io.github.aliyunmq ++ org.apache.rocketmq + rocketmq-rocksdb + + +diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java +index 1233a5422..407ef2842 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java ++++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java +@@ -492,6 +492,7 @@ public class MixAll { + public static int compareLong(long x, long y) { + return Long.compare(x, y); + } ++ + public static boolean isLmq(String lmqMetaData) { + return lmqMetaData != null && lmqMetaData.startsWith(LMQ_PREFIX); + } +diff --git a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java +index 73ef21880..9148d5a18 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java ++++ b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java +@@ -19,5 +19,6 @@ package org.apache.rocketmq.common.attribute; + + public enum CQType { + SimpleCQ, +- BatchCQ ++ BatchCQ, ++ RocksDBCQ + } +diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +index 6f19a9815..20319abba 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +@@ -17,7 +17,6 @@ + package org.apache.rocketmq.common.config; + + import java.nio.ByteBuffer; +-import java.nio.charset.Charset; + import java.util.ArrayList; + import java.util.List; + import java.util.Map; +@@ -27,11 +26,11 @@ import java.util.concurrent.Semaphore; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + +-import com.google.common.collect.Lists; + import com.google.common.collect.Maps; + + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.DataConverter; + import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +@@ -47,7 +46,6 @@ import org.rocksdb.Priority; + import org.rocksdb.ReadOptions; + import org.rocksdb.RocksDB; + import org.rocksdb.RocksDBException; +-import org.rocksdb.RocksIterator; + import org.rocksdb.Statistics; + import org.rocksdb.Status; + import org.rocksdb.WriteBatch; +@@ -58,7 +56,6 @@ import static org.rocksdb.RocksDB.NOT_FOUND; + public abstract class AbstractRocksDBStorage { + protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + +- private static final Charset CHARSET_UTF8 = Charset.forName("UTF-8"); + private static final String SPACE = " | "; + + protected String dbPath; +@@ -223,10 +220,6 @@ public abstract class AbstractRocksDBStorage { + } + } + +- protected WrappedRocksIterator newIterator(ColumnFamilyHandle cfHandle, ReadOptions readOptions) { +- return new WrappedRocksIterator(this.db.newIterator(cfHandle, readOptions)); +- } +- + protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, + final byte[] startKey, final byte[] endKey) throws RocksDBException { + if (!hold()) { +@@ -243,46 +236,6 @@ public abstract class AbstractRocksDBStorage { + } + } + +- protected void manualCompactionDefaultCfMaxLevel(final CompactionOptions compactionOptions) throws Exception { +- final ColumnFamilyHandle defaultCFHandle = this.defaultCFHandle; +- final byte[] defaultCFName = defaultCFHandle.getName(); +- List fileMetaDataList = this.db.getLiveFilesMetaData(); +- if (fileMetaDataList == null || fileMetaDataList.isEmpty()) { +- return; +- } +- +- List defaultLiveFileDataList = Lists.newArrayList(); +- List inputFileNames = Lists.newArrayList(); +- int maxLevel = 0; +- for (LiveFileMetaData fileMetaData : fileMetaDataList) { +- if (compareTo(fileMetaData.columnFamilyName(), defaultCFName) != 0) { +- continue; +- } +- defaultLiveFileDataList.add(fileMetaData); +- if (fileMetaData.level() > maxLevel) { +- maxLevel = fileMetaData.level(); +- } +- } +- if (maxLevel == 0) { +- LOGGER.info("manualCompactionDefaultCfFiles skip level 0."); +- return; +- } +- +- for (LiveFileMetaData fileMetaData : defaultLiveFileDataList) { +- if (fileMetaData.level() != maxLevel || fileMetaData.beingCompacted()) { +- continue; +- } +- inputFileNames.add(fileMetaData.path() + fileMetaData.fileName()); +- } +- if (!inputFileNames.isEmpty()) { +- List outputLists = this.db.compactFiles(compactionOptions, defaultCFHandle, +- inputFileNames, maxLevel, -1, null); +- LOGGER.info("manualCompactionDefaultCfFiles OK. src: {}, dst: {}", inputFileNames, outputLists); +- } else { +- LOGGER.info("manualCompactionDefaultCfFiles Empty."); +- } +- } +- + protected void manualCompactionDefaultCfRange(CompactRangeOptions compactRangeOptions) { + if (!hold()) { + return; +@@ -494,50 +447,6 @@ public abstract class AbstractRocksDBStorage { + this.db.flushWal(true); + } + +- protected class WrappedRocksIterator { +- private final RocksIterator iterator; +- +- public WrappedRocksIterator(final RocksIterator iterator) { +- this.iterator = iterator; +- } +- +- public byte[] key() { +- return iterator.key(); +- } +- +- public byte[] value() { +- return iterator.value(); +- } +- +- public void next() { +- iterator.next(); +- } +- +- public void prev() { +- iterator.prev(); +- } +- +- public void seek(byte[] target) { +- iterator.seek(target); +- } +- +- public void seekForPrev(byte[] target) { +- iterator.seekForPrev(target); +- } +- +- public void seekToFirst() { +- iterator.seekToFirst(); +- } +- +- public boolean isValid() { +- return iterator.isValid(); +- } +- +- public void close() { +- iterator.close(); +- } +- } +- + private String getStatusError(RocksDBException e) { + if (e == null || e.getStatus() == null) { + return "null"; +@@ -574,7 +483,7 @@ public abstract class AbstractRocksDBStorage { + sb = new StringBuilder(256); + map.put(metaData.level(), sb); + } +- sb.append(new String(metaData.columnFamilyName(), CHARSET_UTF8)).append(SPACE). ++ sb.append(new String(metaData.columnFamilyName(), DataConverter.CHARSET_UTF8)).append(SPACE). + append(metaData.fileName()).append(SPACE). + append("s: ").append(metaData.size()).append(SPACE). + append("a: ").append(metaData.numEntries()).append(SPACE). +@@ -595,21 +504,4 @@ public abstract class AbstractRocksDBStorage { + } catch (Exception ignored) { + } + } +- +- public int compareTo(byte[] v1, byte[] v2) { +- int len1 = v1.length; +- int len2 = v2.length; +- int lim = Math.min(len1, len2); +- +- int k = 0; +- while (k < lim) { +- byte c1 = v1[k]; +- byte c2 = v2[k]; +- if (c1 != c2) { +- return c1 - c2; +- } +- k++; +- } +- return len1 - len2; +- } + } +\ No newline at end of file +diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +index 463bd8fed..b40f8046e 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java ++++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +@@ -203,7 +203,7 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage { + setUseDirectReads(true); + } + +- private static String getDBLogDir() { ++ public static String getDBLogDir() { + String rootPath = System.getProperty("user.home"); + if (StringUtils.isEmpty(rootPath)) { + return ""; +diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java +index 0c72ebb7b..91599653c 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java ++++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java +@@ -70,4 +70,17 @@ public class MessageExtBrokerInner extends MessageExt { + public void setVersion(MessageVersion version) { + this.version = version; + } ++ ++ public void removeWaitStorePropertyString() { ++ if (this.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) { ++ // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message. ++ // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it. ++ String waitStoreMsgOKValue = this.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); ++ this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties())); ++ // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later ++ this.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue); ++ } else { ++ this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties())); ++ } ++ } + } +diff --git a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java +index 61265c05d..c19592a44 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java ++++ b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java +@@ -31,6 +31,7 @@ public class TopicValidator { + public static final String RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC = "TRANS_CHECK_MAX_TIME_TOPIC"; + public static final String RMQ_SYS_SELF_TEST_TOPIC = "SELF_TEST_TOPIC"; + public static final String RMQ_SYS_OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT"; ++ public static final String RMQ_SYS_ROCKSDB_OFFSET_TOPIC = "CHECKPOINT_TOPIC"; + + public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; + public static final String SYNC_BROKER_MEMBER_GROUP_PREFIX = SYSTEM_TOPIC_PREFIX + "SYNC_BROKER_MEMBER_"; +@@ -55,6 +56,7 @@ public class TopicValidator { + SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_SELF_TEST_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_OFFSET_MOVED_EVENT); ++ SYSTEM_TOPIC_SET.add(RMQ_SYS_ROCKSDB_OFFSET_TOPIC); + + NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_SCHEDULE_TOPIC); + NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_TRANS_HALF_TOPIC); +diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java +index 8b50de12b..cc96770b2 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java ++++ b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java +@@ -20,7 +20,7 @@ import java.nio.ByteBuffer; + import java.nio.charset.Charset; + + public class DataConverter { +- public static Charset charset = Charset.forName("UTF-8"); ++ public static final Charset CHARSET_UTF8 = Charset.forName("UTF-8"); + + public static byte[] Long2Byte(Long v) { + ByteBuffer tmp = ByteBuffer.allocate(8); +diff --git a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java +index aac295030..8662328ea 100644 +--- a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java ++++ b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java +@@ -71,7 +71,7 @@ public class Producer { + TAG /* Tag */, + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */ + ); +- ++ msg.setDelayTimeLevel(1); + /* + * Call send message to deliver message to one of brokers. + */ +diff --git a/metrics b/metrics +new file mode 100644 +index 000000000..146e90fae +--- /dev/null ++++ b/metrics +@@ -0,0 +1,241 @@ ++# TYPE target info ++# HELP target Target metadata ++target_info{} 1 ++# TYPE otel_scope_info info ++# HELP otel_scope_info Scope metadata ++otel_scope_info{otel_scope_name="broker-meter"} 1 ++# TYPE rocketmq_storage_message_reserve_time gauge ++# HELP rocketmq_storage_message_reserve_time default view for gauge. ++rocketmq_storage_message_reserve_time_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1.9491132515E10 1696905287361 ++# TYPE rocketmq_timer_enqueue_latency gauge ++# HELP rocketmq_timer_enqueue_latency default view for gauge. ++rocketmq_timer_enqueue_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_timer_enqueue_lag gauge ++# HELP rocketmq_timer_enqueue_lag default view for gauge. ++rocketmq_timer_enqueue_lag{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_timing_messages gauge ++# HELP rocketmq_timing_messages default view for gauge. ++rocketmq_timing_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local",topic="BenchmarkTest"} 0.0 1696905287361 ++# TYPE rocketmq_storage_flush_behind_bytes gauge ++# HELP rocketmq_storage_flush_behind_bytes default view for gauge. ++rocketmq_storage_flush_behind_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1230.0 1696905287361 ++# TYPE rocketmq_processor_watermark gauge ++# HELP rocketmq_processor_watermark default view for gauge. ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="reply"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="transaction"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="send"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="async_put"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="lite_pull"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="pull"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="heartbeat"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="consumer_manager"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="admin"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="client_manager"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="query_message"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="ack"} 0.0 1696905287361 ++# TYPE rocketmq_pop_buffer_scan_time_consume histogram ++# HELP rocketmq_pop_buffer_scan_time_consume Time consuming of pop buffer scan ++rocketmq_pop_buffer_scan_time_consume_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 5267005.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 39651.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="1.0"} 5266935.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="10.0"} 5266998.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="100.0"} 5267001.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="1000.0"} 5267005.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="2000.0"} 5267005.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="3000.0"} 5267005.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="+Inf"} 5267005.0 1696905287361 ++# TYPE rocketmq_pop_revive_lag gauge ++# HELP rocketmq_pop_revive_lag default view for gauge. ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="1"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="2"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="3"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="4"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="5"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="6"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="7"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="0"} 1.0 1696905287361 ++# TYPE rocketmq_pop_revive_latency gauge ++# HELP rocketmq_pop_revive_latency default view for gauge. ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="1"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="2"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="3"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="4"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="5"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="6"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="7"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="0"} 0.0 1696905287361 ++# TYPE rocketmq_storage_size gauge ++# HELP rocketmq_storage_size default view for gauge. ++rocketmq_storage_size_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1.5905103872E10 1696905287361 ++# TYPE rocketmq_pop_offset_buffer_size gauge ++# HELP rocketmq_pop_offset_buffer_size default view for gauge. ++rocketmq_pop_offset_buffer_size{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 0.0 1696905287361 ++# TYPE rocketmq_timer_dequeue_latency gauge ++# HELP rocketmq_timer_dequeue_latency default view for gauge. ++rocketmq_timer_dequeue_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_broker_permission gauge ++# HELP rocketmq_broker_permission default view for gauge. ++rocketmq_broker_permission{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 6.0 1696905287361 ++# TYPE rocketmq_consumer_lag_messages gauge ++# HELP rocketmq_consumer_lag_messages default view for gauge. ++rocketmq_consumer_lag_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_throughput_out_total counter ++# HELP rocketmq_throughput_out_total default view for counter. ++rocketmq_throughput_out_total{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361 ++# TYPE rocketmq_timer_dequeue_lag gauge ++# HELP rocketmq_timer_dequeue_lag default view for gauge. ++rocketmq_timer_dequeue_lag{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_messages_out_total counter ++# HELP rocketmq_messages_out_total default view for counter. ++rocketmq_messages_out_total{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361 ++# TYPE rocketmq_messages_in_total counter ++# HELP rocketmq_messages_in_total default view for counter. ++rocketmq_messages_in_total{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361 ++# TYPE rocketmq_consumer_queueing_latency gauge ++# HELP rocketmq_consumer_queueing_latency default view for gauge. ++rocketmq_consumer_queueing_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_consumer_connections gauge ++# HELP rocketmq_consumer_connections default view for gauge. ++rocketmq_consumer_connections{otel_scope_name="broker-meter",cluster="DefaultCluster",consume_mode="push",consumer_group="MQ_INST_test%GID_222",is_system="false",language="java",node_id="broker-a",node_type="broker",protocol_type="remoting",version="v4_8_0"} 1.0 1696905287361 ++# TYPE rocketmq_rpc_latency histogram ++# HELP rocketmq_rpc_latency Rpc latency ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success"} 128.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="1.0"} 17.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="3.0"} 18.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="5.0"} 19.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="7.0"} 19.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="10.0"} 20.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="100.0"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="1000.0"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="2000.0"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="3000.0"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="+Inf"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success"} 8.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="1.0"} 3.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="3.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="5.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="7.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="10.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="100.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="1000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="2000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="3000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="+Inf"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="1.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="3.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="5.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="7.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="10.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="100.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="1000.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="2000.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="3000.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="+Inf"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success"} 108.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="1.0"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="3.0"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="5.0"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="7.0"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="10.0"} 55.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="100.0"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="1000.0"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="2000.0"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="3000.0"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="+Inf"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success"} 210.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="1.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="3.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="5.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="7.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="10.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="100.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="1000.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="2000.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="3000.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="+Inf"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success"} 155.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success"} 2811756.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="1.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="3.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="5.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="7.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="10.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="100.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="1000.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="2000.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="3000.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="+Inf"} 155.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success"} 225.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="1.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="3.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="5.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="7.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="10.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="100.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="1000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="2000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="3000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="+Inf"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway"} 84.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="1.0"} 551.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="3.0"} 557.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="5.0"} 558.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="7.0"} 560.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="10.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="100.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="1000.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="2000.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="3000.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="+Inf"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success"} 6.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="1.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="3.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="5.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="7.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="10.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="100.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="1000.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="2000.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="3000.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="+Inf"} 2.0 1696905287361 ++# TYPE rocketmq_storage_dispatch_behind_bytes gauge ++# HELP rocketmq_storage_dispatch_behind_bytes default view for gauge. ++rocketmq_storage_dispatch_behind_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_consumer_ready_messages gauge ++# HELP rocketmq_consumer_ready_messages default view for gauge. ++rocketmq_consumer_ready_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_consumer_lag_latency gauge ++# HELP rocketmq_consumer_lag_latency default view for gauge. ++rocketmq_consumer_lag_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_throughput_in_total counter ++# HELP rocketmq_throughput_in_total default view for counter. ++rocketmq_throughput_in_total{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361 ++# TYPE rocketmq_pop_checkpoint_buffer_size gauge ++# HELP rocketmq_pop_checkpoint_buffer_size default view for gauge. ++rocketmq_pop_checkpoint_buffer_size{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 0.0 1696905287361 ++# TYPE rocketmq_message_size histogram ++# HELP rocketmq_message_size Incoming messages size ++rocketmq_message_size_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361 ++rocketmq_message_size_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="1024.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="4096.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="524288.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="1048576.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="2097152.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="4194304.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="+Inf"} 5.0 1696905287361 ++# TYPE rocketmq_consumer_inflight_messages gauge ++# HELP rocketmq_consumer_inflight_messages default view for gauge. ++rocketmq_consumer_inflight_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_producer_connections gauge ++# HELP rocketmq_producer_connections default view for gauge. ++rocketmq_producer_connections{otel_scope_name="broker-meter",cluster="DefaultCluster",language="java",node_id="broker-a",node_type="broker",protocol_type="remoting",version="v4_8_0"} 1.0 1696905287361 +diff --git a/patch009-modify-CRLF-to-LF.patch b/patch009-modify-CRLF-to-LF.patch +new file mode 100644 +index 000000000..c9f1f1843 +--- /dev/null ++++ b/patch009-modify-CRLF-to-LF.patch +@@ -0,0 +1,127 @@ ++From 81cd734ee33586544694675bd1081f2c81024d22 Mon Sep 17 00:00:00 2001 ++From: zhiliatom ++Date: Thu, 28 Sep 2023 09:58:13 +0800 ++Subject: [PATCH] modify line separater from CRLF to LF ++ ++--- ++ .../rocketmq/common/constant/LoggerName.java | 110 +++++++++--------- ++ 1 file changed, 55 insertions(+), 55 deletions(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++index cb04b00b3..5c7afab9f 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +++++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++@@ -1,55 +1,55 @@ ++-/* ++- * Licensed to the Apache Software Foundation (ASF) under one or more ++- * contributor license agreements. See the NOTICE file distributed with ++- * this work for additional information regarding copyright ownership. ++- * The ASF licenses this file to You under the Apache License, Version 2.0 ++- * (the "License"); you may not use this file except in compliance with ++- * the License. You may obtain a copy of the License at ++- * ++- * http://www.apache.org/licenses/LICENSE-2.0 ++- * ++- * Unless required by applicable law or agreed to in writing, software ++- * distributed under the License is distributed on an "AS IS" BASIS, ++- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++- * See the License for the specific language governing permissions and ++- * limitations under the License. ++- */ ++-package org.apache.rocketmq.common.constant; ++- ++-public class LoggerName { ++- public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv"; ++- public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; ++- public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; ++- public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; ++- public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; ++- public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; ++- public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; ++- public static final String CLIENT_LOGGER_NAME = "RocketmqClient"; ++- public static final String ROCKETMQ_TRAFFIC_NAME = "RocketmqTraffic"; ++- public static final String ROCKETMQ_REMOTING_NAME = "RocketmqRemoting"; ++- public static final String TOOLS_LOGGER_NAME = "RocketmqTools"; ++- public static final String COMMON_LOGGER_NAME = "RocketmqCommon"; ++- public static final String STORE_LOGGER_NAME = "RocketmqStore"; ++- public static final String STORE_ERROR_LOGGER_NAME = "RocketmqStoreError"; ++- public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction"; ++- public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock"; ++- public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats"; ++- public static final String DLQ_STATS_LOGGER_NAME = "RocketmqDLQStats"; ++- public static final String DLQ_LOGGER_NAME = "RocketmqDLQ"; ++- public static final String CONSUMER_STATS_LOGGER_NAME = "RocketmqConsumerStats"; ++- public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial"; ++- public static final String ACCOUNT_LOGGER_NAME = "RocketmqAccount"; ++- public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl"; ++- public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize"; ++- public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication"; ++- public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection"; ++- public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark"; ++- public static final String FILTER_LOGGER_NAME = "RocketmqFilter"; ++- public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop"; ++- public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover"; ++- public static final String STDOUT_LOGGER_NAME = "STDOUT"; ++- public static final String PROXY_LOGGER_NAME = "RocketmqProxy"; ++- public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark"; ++- public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = "RocketmqColdCtr"; ++- public static final String ROCKSDB_LOGGER_NAME = "RocketmqRocksDB"; ++-} +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.common.constant; +++ +++public class LoggerName { +++ public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv"; +++ public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; +++ public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; +++ public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; +++ public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; +++ public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; +++ public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; +++ public static final String CLIENT_LOGGER_NAME = "RocketmqClient"; +++ public static final String ROCKETMQ_TRAFFIC_NAME = "RocketmqTraffic"; +++ public static final String ROCKETMQ_REMOTING_NAME = "RocketmqRemoting"; +++ public static final String TOOLS_LOGGER_NAME = "RocketmqTools"; +++ public static final String COMMON_LOGGER_NAME = "RocketmqCommon"; +++ public static final String STORE_LOGGER_NAME = "RocketmqStore"; +++ public static final String STORE_ERROR_LOGGER_NAME = "RocketmqStoreError"; +++ public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction"; +++ public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock"; +++ public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats"; +++ public static final String DLQ_STATS_LOGGER_NAME = "RocketmqDLQStats"; +++ public static final String DLQ_LOGGER_NAME = "RocketmqDLQ"; +++ public static final String CONSUMER_STATS_LOGGER_NAME = "RocketmqConsumerStats"; +++ public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial"; +++ public static final String ACCOUNT_LOGGER_NAME = "RocketmqAccount"; +++ public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl"; +++ public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize"; +++ public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication"; +++ public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection"; +++ public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark"; +++ public static final String FILTER_LOGGER_NAME = "RocketmqFilter"; +++ public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop"; +++ public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover"; +++ public static final String STDOUT_LOGGER_NAME = "STDOUT"; +++ public static final String PROXY_LOGGER_NAME = "RocketmqProxy"; +++ public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark"; +++ public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = "RocketmqColdCtr"; +++ public static final String ROCKSDB_LOGGER_NAME = "RocketmqRocksDB"; +++} ++-- ++2.32.0.windows.2 ++ +diff --git a/patch010-backport-add-some-fixes b/patch010-backport-add-some-fixes +new file mode 100644 +index 000000000..36d5e4a1f +--- /dev/null ++++ b/patch010-backport-add-some-fixes +@@ -0,0 +1,1286 @@ ++From b2deef179dbc6a9eb1a2b6dd7b652d95cb768295 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Thu, 10 Aug 2023 10:38:47 +0800 ++Subject: [PATCH 01/12] [ISSUE #7144] Accelerate the recovery speed of the ++ tiered storage module (#7145) ++ ++--- ++ .../tieredstore/TieredDispatcher.java | 3 + ++ .../tieredstore/TieredMessageStore.java | 2 +- ++ .../common/TieredStoreExecutor.java | 25 ++-- ++ .../tieredstore/file/CompositeFlatFile.java | 15 +- ++ .../file/CompositeQueueFlatFile.java | 20 ++- ++ .../tieredstore/file/TieredCommitLog.java | 24 +++- ++ .../tieredstore/file/TieredFlatFile.java | 42 +++--- ++ .../file/TieredFlatFileManager.java | 135 ++++++++++-------- ++ .../metadata/FileSegmentMetadata.java | 26 +++- ++ .../tieredstore/TieredDispatcherTest.java | 15 +- ++ .../tieredstore/TieredMessageFetcherTest.java | 2 +- ++ .../file/CompositeQueueFlatFileTest.java | 2 +- ++ .../file/TieredFlatFileManagerTest.java | 7 +- ++ 13 files changed, 194 insertions(+), 124 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++index bb58ea7dd..1746190cd 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++@@ -279,6 +279,9 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ long upperBound = Math.min(dispatchOffset + maxCount, maxOffsetInQueue); ++ ConsumeQueue consumeQueue = (ConsumeQueue) defaultStore.getConsumeQueue(topic, queueId); ++ +++ logger.debug("DispatchFlatFile race, topic={}, queueId={}, cq range={}-{}, dispatch offset={}-{}", +++ topic, queueId, minOffsetInQueue, maxOffsetInQueue, dispatchOffset, upperBound - 1); +++ ++ for (; dispatchOffset < upperBound; dispatchOffset++) { ++ // get consume queue ++ SelectMappedBufferResult cqItem = consumeQueue.getIndexBuffer(dispatchOffset); ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index 1f12410f2..ced1fb818 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -147,7 +147,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { ++ ++ if (!viaTieredStorage(topic, queueId, offset, maxMsgNums)) { ++- logger.debug("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); +++ logger.trace("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); ++ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); ++ } ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++index 6eb3478b3..6dd0e8846 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++@@ -43,18 +43,9 @@ public class TieredStoreExecutor { ++ public static ExecutorService compactIndexFileExecutor; ++ ++ public static void init() { ++- dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++- dispatchExecutor = new ThreadPoolExecutor( ++- Math.max(2, Runtime.getRuntime().availableProcessors()), ++- Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++- 1000 * 60, ++- TimeUnit.MILLISECONDS, ++- dispatchThreadPoolQueue, ++- new ThreadFactoryImpl("TieredCommonExecutor_")); ++- ++ commonScheduledExecutor = new ScheduledThreadPoolExecutor( ++ Math.max(4, Runtime.getRuntime().availableProcessors()), ++- new ThreadFactoryImpl("TieredCommonScheduledExecutor_")); +++ new ThreadFactoryImpl("TieredCommonExecutor_")); ++ ++ commitExecutor = new ScheduledThreadPoolExecutor( ++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++@@ -62,7 +53,17 @@ public class TieredStoreExecutor { ++ ++ cleanExpiredFileExecutor = new ScheduledThreadPoolExecutor( ++ Math.max(4, Runtime.getRuntime().availableProcessors()), ++- new ThreadFactoryImpl("TieredCleanExpiredFileExecutor_")); +++ new ThreadFactoryImpl("TieredCleanFileExecutor_")); +++ +++ dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); +++ dispatchExecutor = new ThreadPoolExecutor( +++ Math.max(2, Runtime.getRuntime().availableProcessors()), +++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), +++ 1000 * 60, +++ TimeUnit.MILLISECONDS, +++ dispatchThreadPoolQueue, +++ new ThreadFactoryImpl("TieredDispatchExecutor_"), +++ new ThreadPoolExecutor.DiscardOldestPolicy()); ++ ++ fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++ fetchDataExecutor = new ThreadPoolExecutor( ++@@ -71,7 +72,7 @@ public class TieredStoreExecutor { ++ 1000 * 60, ++ TimeUnit.MILLISECONDS, ++ fetchDataThreadPoolQueue, ++- new ThreadFactoryImpl("TieredFetchDataExecutor_")); +++ new ThreadFactoryImpl("TieredFetchExecutor_")); ++ ++ compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++ compactIndexFileExecutor = new ThreadPoolExecutor( ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java ++index df4baf33f..5ad3a6ff3 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java ++@@ -76,20 +76,15 @@ public class CompositeFlatFile implements CompositeAccess { ++ this.storeConfig = fileQueueFactory.getStoreConfig(); ++ this.readAheadFactor = this.storeConfig.getReadAheadMinFactor(); ++ this.metadataStore = TieredStoreUtil.getMetadataStore(this.storeConfig); ++- this.dispatchOffset = new AtomicLong(); ++ this.compositeFlatFileLock = new ReentrantLock(); ++ this.inFlightRequestMap = new ConcurrentHashMap<>(); ++ this.commitLog = new TieredCommitLog(fileQueueFactory, filePath); ++ this.consumeQueue = new TieredConsumeQueue(fileQueueFactory, filePath); +++ this.dispatchOffset = new AtomicLong( +++ this.consumeQueue.isInitialized() ? this.getConsumeQueueCommitOffset() : -1L); ++ this.groupOffsetCache = this.initOffsetCache(); ++ } ++ ++- protected void recoverMetadata() { ++- if (!consumeQueue.isInitialized() && this.dispatchOffset.get() != -1) { ++- consumeQueue.setBaseOffset(this.dispatchOffset.get() * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE); ++- } ++- } ++- ++ private Cache initOffsetCache() { ++ return Caffeine.newBuilder() ++ .expireAfterWrite(2, TimeUnit.MINUTES) ++@@ -310,10 +305,12 @@ public class CompositeFlatFile implements CompositeAccess { ++ ++ @Override ++ public void initOffset(long offset) { ++- if (!consumeQueue.isInitialized()) { +++ if (consumeQueue.isInitialized()) { +++ dispatchOffset.set(this.getConsumeQueueCommitOffset()); +++ } else { ++ consumeQueue.setBaseOffset(offset * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE); +++ dispatchOffset.set(offset); ++ } ++- dispatchOffset.set(offset); ++ } ++ ++ @Override ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java ++index f6c0afed0..0a797f465 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java ++@@ -36,8 +36,7 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { ++ public CompositeQueueFlatFile(TieredFileAllocator fileQueueFactory, MessageQueue messageQueue) { ++ super(fileQueueFactory, TieredStoreUtil.toPath(messageQueue)); ++ this.messageQueue = messageQueue; ++- this.recoverTopicMetadata(); ++- super.recoverMetadata(); +++ this.recoverQueueMetadata(); ++ this.indexFile = TieredFlatFileManager.getIndexFile(storeConfig); ++ } ++ ++@@ -46,11 +45,12 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { ++ if (!consumeQueue.isInitialized()) { ++ queueMetadata.setMinOffset(offset); ++ queueMetadata.setMaxOffset(offset); +++ metadataStore.updateQueue(queueMetadata); ++ } ++ super.initOffset(offset); ++ } ++ ++- public void recoverTopicMetadata() { +++ public void recoverQueueMetadata() { ++ TopicMetadata topicMetadata = this.metadataStore.getTopic(messageQueue.getTopic()); ++ if (topicMetadata == null) { ++ topicMetadata = this.metadataStore.addTopic(messageQueue.getTopic(), -1L); ++@@ -64,18 +64,16 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { ++ if (queueMetadata.getMaxOffset() < queueMetadata.getMinOffset()) { ++ queueMetadata.setMaxOffset(queueMetadata.getMinOffset()); ++ } ++- this.dispatchOffset.set(queueMetadata.getMaxOffset()); ++ } ++ ++- public void persistMetadata() { +++ public void flushMetadata() { ++ try { ++- if (consumeQueue.getCommitOffset() < queueMetadata.getMinOffset()) { ++- return; ++- } ++- queueMetadata.setMaxOffset(consumeQueue.getCommitOffset() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE); +++ queueMetadata.setMinOffset(super.getConsumeQueueMinOffset()); +++ queueMetadata.setMaxOffset(super.getConsumeQueueMaxOffset()); ++ metadataStore.updateQueue(queueMetadata); ++ } catch (Exception e) { ++- LOGGER.error("CompositeFlatFile#flushMetadata: update queue metadata failed: topic: {}, queue: {}", messageQueue.getTopic(), messageQueue.getQueueId(), e); +++ LOGGER.error("CompositeFlatFile#flushMetadata error, topic: {}, queue: {}", +++ messageQueue.getTopic(), messageQueue.getQueueId(), e); ++ } ++ } ++ ++@@ -114,7 +112,7 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { ++ @Override ++ public void shutdown() { ++ super.shutdown(); ++- metadataStore.updateQueue(queueMetadata); +++ this.flushMetadata(); ++ } ++ ++ @Override ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java ++index 80e1bce50..0e5f79132 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java ++@@ -50,7 +50,7 @@ public class TieredCommitLog { ++ this.storeConfig = fileQueueFactory.getStoreConfig(); ++ this.flatFile = fileQueueFactory.createFlatFileForCommitLog(filePath); ++ this.minConsumeQueueOffset = new AtomicLong(NOT_EXIST_MIN_OFFSET); ++- this.correctMinOffset(); +++ this.correctMinOffsetAsync(); ++ } ++ ++ @VisibleForTesting ++@@ -91,17 +91,26 @@ public class TieredCommitLog { ++ return flatFile.getFileToWrite().getMaxTimestamp(); ++ } ++ ++- public synchronized long correctMinOffset() { +++ public long correctMinOffset() { +++ try { +++ return correctMinOffsetAsync().get(); +++ } catch (Exception e) { +++ log.error("Correct min offset failed in clean expired file", e); +++ } +++ return NOT_EXIST_MIN_OFFSET; +++ } +++ +++ public synchronized CompletableFuture correctMinOffsetAsync() { ++ if (flatFile.getFileSegmentCount() == 0) { ++ this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET); ++- return NOT_EXIST_MIN_OFFSET; +++ return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET); ++ } ++ ++ // queue offset field length is 8 ++ int length = MessageBufferUtil.QUEUE_OFFSET_POSITION + 8; ++ if (flatFile.getCommitOffset() - flatFile.getMinOffset() < length) { ++ this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET); ++- return NOT_EXIST_MIN_OFFSET; +++ return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET); ++ } ++ ++ try { ++@@ -109,7 +118,8 @@ public class TieredCommitLog { ++ .thenApply(buffer -> { ++ long offset = MessageBufferUtil.getQueueOffset(buffer); ++ minConsumeQueueOffset.set(offset); ++- log.info("Correct commitlog min cq offset success, filePath={}, min cq offset={}, range={}-{}", +++ log.debug("Correct commitlog min cq offset success, " + +++ "filePath={}, min cq offset={}, commitlog range={}-{}", ++ flatFile.getFilePath(), offset, flatFile.getMinOffset(), flatFile.getCommitOffset()); ++ return offset; ++ }) ++@@ -117,11 +127,11 @@ public class TieredCommitLog { ++ log.warn("Correct commitlog min cq offset error, filePath={}, range={}-{}", ++ flatFile.getFilePath(), flatFile.getMinOffset(), flatFile.getCommitOffset(), throwable); ++ return minConsumeQueueOffset.get(); ++- }).get(); +++ }); ++ } catch (Exception e) { ++ log.error("Correct commitlog min cq offset error, filePath={}", flatFile.getFilePath(), e); ++ } ++- return minConsumeQueueOffset.get(); +++ return CompletableFuture.completedFuture(minConsumeQueueOffset.get()); ++ } ++ ++ public AppendResult append(ByteBuffer byteBuf) { ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++index 75ce8d89f..426c4e09d 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ +++import com.alibaba.fastjson.JSON; ++ import com.google.common.annotations.VisibleForTesting; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; ++@@ -24,6 +25,7 @@ import java.util.Comparator; ++ import java.util.HashSet; ++ import java.util.LinkedList; ++ import java.util.List; +++import java.util.Objects; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.CopyOnWriteArrayList; ++@@ -178,32 +180,26 @@ public class TieredFlatFile { ++ private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fileSegment) { ++ ++ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( ++- fileSegment.getPath(), fileSegment.getFileType(), fileSegment.getBaseOffset()); ++- ++- if (metadata != null) { ++- return metadata; ++- } +++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); ++ ++ // Note: file segment path may not the same as file base path, use base path here. ++- metadata = new FileSegmentMetadata( ++- this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); ++- ++- if (fileSegment.isClosed()) { ++- metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); +++ if (metadata == null) { +++ metadata = new FileSegmentMetadata( +++ this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); +++ metadata.setCreateTimestamp(fileSegment.getMinTimestamp()); +++ metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); +++ metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); +++ if (fileSegment.isClosed()) { +++ metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); +++ } +++ this.tieredMetadataStore.updateFileSegment(metadata); ++ } ++- ++- metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); ++- metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++- ++- // Submit to persist ++- this.tieredMetadataStore.updateFileSegment(metadata); ++ return metadata; ++ } ++ ++ /** ++ * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full ++ */ ++- @VisibleForTesting ++ public void updateFileSegment(TieredFileSegment fileSegment) { ++ FileSegmentMetadata segmentMetadata = getOrCreateFileSegmentMetadata(fileSegment); ++ ++@@ -219,9 +215,16 @@ public class TieredFlatFile { ++ } ++ ++ segmentMetadata.setSize(fileSegment.getCommitPosition()); ++- segmentMetadata.setBeginTimestamp(fileSegment.getMinTimestamp()); ++ segmentMetadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++- this.tieredMetadataStore.updateFileSegment(segmentMetadata); +++ +++ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( +++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); +++ +++ if (!Objects.equals(metadata, segmentMetadata)) { +++ this.tieredMetadataStore.updateFileSegment(segmentMetadata); +++ logger.debug("TieredFlatFile#UpdateSegmentMetadata, filePath: {}, content: {}", +++ segmentMetadata.getPath(), JSON.toJSONString(segmentMetadata)); +++ } ++ } ++ ++ private void checkAndFixFileSize() { ++@@ -257,6 +260,7 @@ public class TieredFlatFile { ++ logger.warn("TieredFlatFile#checkAndFixFileSize: fix last file {} size: origin: {}, actual: {}", ++ lastFile.getPath(), lastFile.getCommitOffset() - lastFile.getBaseOffset(), lastFileSize); ++ lastFile.initPosition(lastFileSize); +++ this.updateFileSegment(lastFile); ++ } ++ } ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++index aeca44b8c..e9ae4a5a5 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++@@ -16,16 +16,19 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ +++import com.google.common.base.Stopwatch; ++ import com.google.common.collect.ImmutableList; ++ import java.util.ArrayList; ++ import java.util.List; ++ import java.util.Random; +++import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.Future; +++import java.util.concurrent.Semaphore; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicLong; ++ import javax.annotation.Nullable; +++import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++@@ -36,6 +39,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ ++ public class TieredFlatFileManager { ++ +++ private static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); ++ private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); ++ ++ private static volatile TieredFlatFileManager instance; ++@@ -44,7 +48,7 @@ public class TieredFlatFileManager { ++ private final TieredMetadataStore metadataStore; ++ private final TieredMessageStoreConfig storeConfig; ++ private final TieredFileAllocator tieredFileAllocator; ++- private final ConcurrentMap queueFlatFileMap; +++ private final ConcurrentMap flatFileConcurrentMap; ++ ++ public TieredFlatFileManager(TieredMessageStoreConfig storeConfig) ++ throws ClassNotFoundException, NoSuchMethodException { ++@@ -52,23 +56,20 @@ public class TieredFlatFileManager { ++ this.storeConfig = storeConfig; ++ this.metadataStore = TieredStoreUtil.getMetadataStore(storeConfig); ++ this.tieredFileAllocator = new TieredFileAllocator(storeConfig); ++- this.queueFlatFileMap = new ConcurrentHashMap<>(); +++ this.flatFileConcurrentMap = new ConcurrentHashMap<>(); ++ this.doScheduleTask(); ++ } ++ ++ public static TieredFlatFileManager getInstance(TieredMessageStoreConfig storeConfig) { ++- if (storeConfig == null) { +++ if (storeConfig == null || instance != null) { ++ return instance; ++ } ++- ++- if (instance == null) { ++- synchronized (TieredFlatFileManager.class) { ++- if (instance == null) { ++- try { ++- instance = new TieredFlatFileManager(storeConfig); ++- } catch (Exception e) { ++- logger.error("TieredFlatFileManager#getInstance: create flat file manager failed", e); ++- } +++ synchronized (TieredFlatFileManager.class) { +++ if (instance == null) { +++ try { +++ instance = new TieredFlatFileManager(storeConfig); +++ } catch (Exception e) { +++ logger.error("Construct FlatFileManager instance error", e); ++ } ++ } ++ } ++@@ -88,7 +89,7 @@ public class TieredFlatFileManager { ++ TieredStoreUtil.RMQ_SYS_TIERED_STORE_INDEX_TOPIC, storeConfig.getBrokerName(), 0)); ++ indexFile = new TieredIndexFile(new TieredFileAllocator(storeConfig), filePath); ++ } catch (Exception e) { ++- logger.error("TieredFlatFileManager#getIndexFile: create index file failed", e); +++ logger.error("Construct FlatFileManager indexFile error", e); ++ } ++ } ++ } ++@@ -105,7 +106,7 @@ public class TieredFlatFileManager { ++ flatFile.commitCommitLog(); ++ } catch (Throwable e) { ++ MessageQueue mq = flatFile.getMessageQueue(); ++- logger.error("commit commitLog periodically failed: topic: {}, queue: {}", +++ logger.error("Commit commitLog periodically failed: topic: {}, queue: {}", ++ mq.getTopic(), mq.getQueueId(), e); ++ } ++ }, delay, TimeUnit.MILLISECONDS); ++@@ -114,7 +115,7 @@ public class TieredFlatFileManager { ++ flatFile.commitConsumeQueue(); ++ } catch (Throwable e) { ++ MessageQueue mq = flatFile.getMessageQueue(); ++- logger.error("commit consumeQueue periodically failed: topic: {}, queue: {}", +++ logger.error("Commit consumeQueue periodically failed: topic: {}, queue: {}", ++ mq.getTopic(), mq.getQueueId(), e); ++ } ++ }, delay, TimeUnit.MILLISECONDS); ++@@ -125,7 +126,7 @@ public class TieredFlatFileManager { ++ indexFile.commit(true); ++ } ++ } catch (Throwable e) { ++- logger.error("commit indexFile periodically failed", e); +++ logger.error("Commit indexFile periodically failed", e); ++ } ++ }, 0, TimeUnit.MILLISECONDS); ++ } ++@@ -160,7 +161,7 @@ public class TieredFlatFileManager { ++ try { ++ doCommit(); ++ } catch (Throwable e) { ++- logger.error("commit flat file periodically failed: ", e); +++ logger.error("Commit flat file periodically failed: ", e); ++ } ++ }, 60, 60, TimeUnit.SECONDS); ++ ++@@ -168,49 +169,73 @@ public class TieredFlatFileManager { ++ try { ++ doCleanExpiredFile(); ++ } catch (Throwable e) { ++- logger.error("clean expired flat file failed: ", e); +++ logger.error("Clean expired flat file failed: ", e); ++ } ++ }, 30, 30, TimeUnit.SECONDS); ++ } ++ ++ public boolean load() { +++ Stopwatch stopwatch = Stopwatch.createStarted(); ++ try { ++- AtomicLong topicSequenceNumber = new AtomicLong(); ++- List> futureList = new ArrayList<>(); ++- queueFlatFileMap.clear(); ++- metadataStore.iterateTopic(topicMetadata -> { +++ flatFileConcurrentMap.clear(); +++ this.recoverSequenceNumber(); +++ this.recoverTieredFlatFile(); +++ logger.info("Message store recover end, total cost={}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); +++ } catch (Exception e) { +++ long costTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); +++ logger.info("Message store recover error, total cost={}ms", costTime); +++ BROKER_LOG.error("Message store recover error, total cost={}ms", costTime, e); +++ return false; +++ } +++ return true; +++ } +++ +++ public void recoverSequenceNumber() { +++ AtomicLong topicSequenceNumber = new AtomicLong(); +++ metadataStore.iterateTopic(topicMetadata -> { +++ if (topicMetadata != null && topicMetadata.getTopicId() > 0) { ++ topicSequenceNumber.set(Math.max(topicSequenceNumber.get(), topicMetadata.getTopicId())); ++- Future future = TieredStoreExecutor.dispatchExecutor.submit(() -> { ++- if (topicMetadata.getStatus() != 0) { ++- return; ++- } +++ } +++ }); +++ metadataStore.setTopicSequenceNumber(topicSequenceNumber.incrementAndGet()); +++ } +++ +++ public void recoverTieredFlatFile() { +++ Semaphore semaphore = new Semaphore((int) (TieredStoreExecutor.QUEUE_CAPACITY * 0.75)); +++ List> futures = new ArrayList<>(); +++ metadataStore.iterateTopic(topicMetadata -> { +++ try { +++ semaphore.acquire(); +++ CompletableFuture future = CompletableFuture.runAsync(() -> { ++ try { ++- metadataStore.iterateQueue(topicMetadata.getTopic(), ++- queueMetadata -> getOrCreateFlatFileIfAbsent( ++- new MessageQueue(topicMetadata.getTopic(), ++- storeConfig.getBrokerName(), ++- queueMetadata.getQueue().getQueueId()))); +++ Stopwatch subWatch = Stopwatch.createStarted(); +++ if (topicMetadata.getStatus() != 0) { +++ return; +++ } +++ AtomicLong queueCount = new AtomicLong(); +++ metadataStore.iterateQueue(topicMetadata.getTopic(), queueMetadata -> { +++ this.getOrCreateFlatFileIfAbsent(new MessageQueue(topicMetadata.getTopic(), +++ storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId())); +++ queueCount.incrementAndGet(); +++ }); +++ logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", +++ topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); ++ } catch (Exception e) { ++- logger.error("load mq composite flat file from metadata failed", e); +++ logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e); +++ } finally { +++ semaphore.release(); ++ } ++- }); ++- futureList.add(future); ++- }); ++- ++- // Wait for load all metadata done ++- for (Future future : futureList) { ++- future.get(); +++ }, TieredStoreExecutor.commitExecutor); +++ futures.add(future); +++ } catch (Exception e) { +++ throw new RuntimeException(e); ++ } ++- metadataStore.setTopicSequenceNumber(topicSequenceNumber.incrementAndGet()); ++- } catch (Exception e) { ++- logger.error("load mq composite flat file from metadata failed", e); ++- return false; ++- } ++- return true; +++ }); +++ CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); ++ } ++ ++ public void cleanup() { ++- queueFlatFileMap.clear(); +++ flatFileConcurrentMap.clear(); ++ cleanStaticReference(); ++ } ++ ++@@ -221,27 +246,25 @@ public class TieredFlatFileManager { ++ ++ @Nullable ++ public CompositeQueueFlatFile getOrCreateFlatFileIfAbsent(MessageQueue messageQueue) { ++- return queueFlatFileMap.computeIfAbsent(messageQueue, mq -> { +++ return flatFileConcurrentMap.computeIfAbsent(messageQueue, mq -> { ++ try { ++- logger.debug("TieredFlatFileManager#getOrCreateFlatFileIfAbsent: " + ++- "try to create new flat file: topic: {}, queueId: {}", +++ logger.debug("Create new TopicFlatFile, topic: {}, queueId: {}", ++ messageQueue.getTopic(), messageQueue.getQueueId()); ++ return new CompositeQueueFlatFile(tieredFileAllocator, mq); ++ } catch (Exception e) { ++- logger.error("TieredFlatFileManager#getOrCreateFlatFileIfAbsent: " + ++- "create new flat file: topic: {}, queueId: {}", +++ logger.debug("Create new TopicFlatFile failed, topic: {}, queueId: {}", ++ messageQueue.getTopic(), messageQueue.getQueueId(), e); ++- return null; ++ } +++ return null; ++ }); ++ } ++ ++ public CompositeQueueFlatFile getFlatFile(MessageQueue messageQueue) { ++- return queueFlatFileMap.get(messageQueue); +++ return flatFileConcurrentMap.get(messageQueue); ++ } ++ ++ public ImmutableList deepCopyFlatFileToList() { ++- return ImmutableList.copyOf(queueFlatFileMap.values()); +++ return ImmutableList.copyOf(flatFileConcurrentMap.values()); ++ } ++ ++ public void shutdown() { ++@@ -270,7 +293,7 @@ public class TieredFlatFileManager { ++ } ++ ++ // delete memory reference ++- CompositeQueueFlatFile flatFile = queueFlatFileMap.remove(mq); +++ CompositeQueueFlatFile flatFile = flatFileConcurrentMap.remove(mq); ++ if (flatFile != null) { ++ MessageQueue messageQueue = flatFile.getMessageQueue(); ++ logger.info("TieredFlatFileManager#destroyCompositeFile: " + ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java ++index 1b232fc75..2f0fd71de 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java ++@@ -16,6 +16,8 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.metadata; ++ +++import java.util.Objects; +++ ++ public class FileSegmentMetadata { ++ ++ public static final int STATUS_NEW = 0; ++@@ -43,7 +45,6 @@ public class FileSegmentMetadata { ++ this.baseOffset = baseOffset; ++ this.type = type; ++ this.status = STATUS_NEW; ++- this.createTimestamp = System.currentTimeMillis(); ++ } ++ ++ public void markSealed() { ++@@ -122,4 +123,27 @@ public class FileSegmentMetadata { ++ public void setSealTimestamp(long sealTimestamp) { ++ this.sealTimestamp = sealTimestamp; ++ } +++ +++ @Override +++ public boolean equals(Object o) { +++ if (this == o) +++ return true; +++ if (o == null || getClass() != o.getClass()) +++ return false; +++ FileSegmentMetadata metadata = (FileSegmentMetadata) o; +++ return size == metadata.size +++ && baseOffset == metadata.baseOffset +++ && status == metadata.status +++ && path.equals(metadata.path) +++ && type == metadata.type +++ && createTimestamp == metadata.createTimestamp +++ && beginTimestamp == metadata.beginTimestamp +++ && endTimestamp == metadata.endTimestamp +++ && sealTimestamp == metadata.sealTimestamp; +++ } +++ +++ @Override +++ public int hashCode() { +++ return Objects.hash(type, path, baseOffset, status, size, createTimestamp, beginTimestamp, endTimestamp, sealTimestamp); +++ } ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java ++index e6adef1d1..5791dc9a4 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java ++@@ -116,19 +116,20 @@ public class TieredDispatcherTest { ++ buffer3.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 9); ++ flatFile.appendCommitLog(buffer3); ++ flatFile.commitCommitLog(); ++- Assert.assertEquals(10, flatFile.getDispatchOffset()); +++ Assert.assertEquals(9 + 1, flatFile.getDispatchOffset()); +++ Assert.assertEquals(9, flatFile.getCommitLogDispatchCommitOffset()); ++ ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer1); ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer2); ++ dispatcher.buildConsumeQueueAndIndexFile(); ++ Assert.assertEquals(7, flatFile.getConsumeQueueMaxOffset()); ++- Assert.assertEquals(7, flatFile.getDispatchOffset()); ++ ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 7, 7, 0, 0, buffer1); ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer2); ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer3); ++ dispatcher.buildConsumeQueueAndIndexFile(); ++- Assert.assertEquals(10, flatFile.getConsumeQueueMaxOffset()); +++ Assert.assertEquals(6, flatFile.getConsumeQueueMinOffset()); +++ Assert.assertEquals(9 + 1, flatFile.getConsumeQueueMaxOffset()); ++ } ++ ++ @Test ++@@ -142,6 +143,7 @@ public class TieredDispatcherTest { ++ Mockito.when(defaultStore.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(0L); ++ Mockito.when(defaultStore.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(9L); ++ +++ // mock cq item, position = 7 ++ ByteBuffer cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE); ++ cqItem.putLong(7); ++ cqItem.putInt(MessageBufferUtilTest.MSG_LEN); ++@@ -150,13 +152,13 @@ public class TieredDispatcherTest { ++ SelectMappedBufferResult mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null); ++ Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(6)).thenReturn(mockResult); ++ +++ // mock cq item, position = 8 ++ cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE); ++ cqItem.putLong(8); ++ cqItem.putInt(MessageBufferUtilTest.MSG_LEN); ++ cqItem.putLong(1); ++ cqItem.flip(); ++ mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null); ++- ++ Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(7)).thenReturn(mockResult); ++ ++ mockResult = new SelectMappedBufferResult(0, MessageBufferUtilTest.buildMockedMessageBuffer(), MessageBufferUtilTest.MSG_LEN, null); ++@@ -167,7 +169,10 @@ public class TieredDispatcherTest { ++ mockResult = new SelectMappedBufferResult(0, msg, MessageBufferUtilTest.MSG_LEN, null); ++ Mockito.when(defaultStore.selectOneMessageByOffset(8, MessageBufferUtilTest.MSG_LEN)).thenReturn(mockResult); ++ ++- dispatcher.dispatchFlatFile(flatFileManager.getOrCreateFlatFileIfAbsent(mq)); +++ CompositeQueueFlatFile flatFile = flatFileManager.getOrCreateFlatFileIfAbsent(mq); +++ Assert.assertNotNull(flatFile); +++ flatFile.initOffset(7); +++ dispatcher.dispatchFlatFile(flatFile); ++ Assert.assertEquals(8, flatFileManager.getFlatFile(mq).getDispatchOffset()); ++ } ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java ++index d75b2f916..774c6cf64 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java ++@@ -23,6 +23,7 @@ import java.util.Objects; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.SystemUtils; ++ import org.apache.commons.lang3.tuple.Triple; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.store.DispatchRequest; ++ import org.apache.rocketmq.store.GetMessageResult; ++@@ -40,7 +41,6 @@ import org.apache.rocketmq.tieredstore.file.TieredIndexFile; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++-import org.apache.rocketmq.common.BoundaryType; ++ import org.awaitility.Awaitility; ++ import org.junit.After; ++ import org.junit.Assert; ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java ++index 27efe111e..2e028ada3 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java ++@@ -119,7 +119,7 @@ public class CompositeQueueFlatFileTest { ++ Assert.assertEquals(AppendResult.SUCCESS, result); ++ ++ file.commit(true); ++- file.persistMetadata(); +++ file.flushMetadata(); ++ ++ QueueMetadata queueMetadata = metadataStore.getQueue(mq); ++ Assert.assertEquals(53, queueMetadata.getMaxOffset()); ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java ++index b7528c5e4..20fe4dd70 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java ++@@ -72,10 +72,15 @@ public class TieredFlatFileManagerTest { ++ ++ CompositeFlatFile flatFile = flatFileManager.getFlatFile(mq); ++ Assert.assertNotNull(flatFile); ++- Assert.assertEquals(100, flatFile.getDispatchOffset()); +++ Assert.assertEquals(-1L, flatFile.getDispatchOffset()); +++ flatFile.initOffset(100L); +++ Assert.assertEquals(100L, flatFile.getDispatchOffset()); +++ flatFile.initOffset(200L); +++ Assert.assertEquals(100L, flatFile.getDispatchOffset()); ++ ++ CompositeFlatFile flatFile1 = flatFileManager.getFlatFile(mq1); ++ Assert.assertNotNull(flatFile1); +++ flatFile1.initOffset(200L); ++ Assert.assertEquals(200, flatFile1.getDispatchOffset()); ++ ++ flatFileManager.destroyCompositeFile(mq); ++-- ++2.32.0.windows.2 ++ ++ ++From 99b39a35f29e491862296d56b7938a995d153974 Mon Sep 17 00:00:00 2001 ++From: ShuangxiDing ++Date: Thu, 10 Aug 2023 11:28:39 +0800 ++Subject: [PATCH 02/12] [ISSUE #7115] Fix grpc response message NPE (#7116) ++ ++--- ++ .../apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java ++index 0b3c85ea6..efa879a9c 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java ++@@ -92,7 +92,7 @@ public class ResponseBuilder { ++ public Status buildStatus(Code code, String message) { ++ return Status.newBuilder() ++ .setCode(code) ++- .setMessage(message) +++ .setMessage(message != null ? message : code.name()) ++ .build(); ++ } ++ ++-- ++2.32.0.windows.2 ++ ++ ++From c0ba453f38183266cf9a69be754e620311e1923b Mon Sep 17 00:00:00 2001 ++From: caigy ++Date: Thu, 10 Aug 2023 14:08:17 +0800 ++Subject: [PATCH 03/12] [ISSUE #7129] Fix resource collisions in acl tests ++ (#7130) ++ ++* run acl tests sequencially to avoid collision ++ ++* disable reuseForks for acl like broker ++ ++* Revert "[ISSUE #7135] Temporarily ignoring plainAccessValidator test (#7135)" ++ ++This reverts commit 6bc2c8474a0ce1e2833c82dffea7b1d8f718fcd7. ++--- ++ acl/pom.xml | 13 +++++++++++++ ++ .../acl/plain/PlainAccessControlFlowTest.java | 5 ----- ++ .../acl/plain/PlainAccessValidatorTest.java | 3 --- ++ .../acl/plain/PlainPermissionManagerTest.java | 3 --- ++ 4 files changed, 13 insertions(+), 11 deletions(-) ++ ++diff --git a/acl/pom.xml b/acl/pom.xml ++index 67bfcb8d2..989c0cf77 100644 ++--- a/acl/pom.xml +++++ b/acl/pom.xml ++@@ -74,4 +74,17 @@ ++ test ++ ++ +++ +++ +++ +++ +++ maven-surefire-plugin +++ ${maven-surefire-plugin.version} +++ +++ 1 +++ false +++ +++ +++ +++ ++ ++diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java ++index e7fd0932f..519345714 100644 ++--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java +++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java ++@@ -31,7 +31,6 @@ import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; ++ import org.junit.Assert; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++ import java.io.File; ++@@ -44,7 +43,6 @@ import java.util.Collections; ++ import java.util.LinkedList; ++ import java.util.List; ++ ++- ++ /** ++ *

In this class, we'll test the following scenarios, each containing several consecutive operations on ACL, ++ *

like updating and deleting ACL, changing config files and checking validations. ++@@ -52,9 +50,6 @@ import java.util.List; ++ *

Case 2: Only conf/acl/plain_acl.yml exists; ++ *

Case 3: Both conf/plain_acl.yml and conf/acl/plain_acl.yml exists. ++ */ ++- ++-// Ignore this test case as it is currently unable to pass on ubuntu workflow ++-@Ignore ++ public class PlainAccessControlFlowTest { ++ public static final String DEFAULT_TOPIC = "topic-acl"; ++ ++diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java ++index a3a925758..ef0cffbdc 100644 ++--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java +++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java ++@@ -56,11 +56,8 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; ++ import org.junit.After; ++ import org.junit.Assert; ++ import org.junit.Before; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++-// Ignore this test case as it is currently unable to pass on ubuntu workflow ++-@Ignore ++ public class PlainAccessValidatorTest { ++ ++ private PlainAccessValidator plainAccessValidator; ++diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java ++index aa7539f3a..941d8c779 100644 ++--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java +++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java ++@@ -29,7 +29,6 @@ import org.assertj.core.api.Assertions; ++ import org.assertj.core.util.Lists; ++ import org.junit.Assert; ++ import org.junit.Before; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++ import java.io.File; ++@@ -42,8 +41,6 @@ import java.util.List; ++ import java.util.Map; ++ import java.util.Set; ++ ++-// Ignore this test case as it is currently unable to pass on ubuntu workflow ++-@Ignore ++ public class PlainPermissionManagerTest { ++ ++ PlainPermissionManager plainPermissionManager; ++-- ++2.32.0.windows.2 ++ ++ ++From 8741ff8c9b3bdbfc97976285affa7ea35c81243c Mon Sep 17 00:00:00 2001 ++From: ShuangxiDing ++Date: Thu, 10 Aug 2023 17:41:15 +0800 ++Subject: [PATCH 04/12] [ISSUE #7153] Add switch for MIXED message type (#7154) ++ ++Add a switch for MIXED message type when creating a Topic in the Broker. ++--- ++ .../broker/processor/AdminBrokerProcessor.java | 8 ++++++++ ++ .../java/org/apache/rocketmq/common/BrokerConfig.java | 10 ++++++++++ ++ 2 files changed, 18 insertions(+) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++index a6ce03dc2..bbddcec2d 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++@@ -59,6 +59,7 @@ import org.apache.rocketmq.common.TopicConfig; ++ import org.apache.rocketmq.common.UnlockCallback; ++ import org.apache.rocketmq.common.UtilAll; ++ import org.apache.rocketmq.common.attribute.AttributeParser; +++import org.apache.rocketmq.common.attribute.TopicMessageType; ++ import org.apache.rocketmq.common.constant.ConsumeInitMode; ++ import org.apache.rocketmq.common.constant.FIleReadaheadMode; ++ import org.apache.rocketmq.common.constant.LoggerName; ++@@ -439,6 +440,13 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ String attributesModification = requestHeader.getAttributes(); ++ topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification)); ++ +++ if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED +++ && !brokerController.getBrokerConfig().isEnableMixedMessageType()) { +++ response.setCode(ResponseCode.SYSTEM_ERROR); +++ response.setRemark("MIXED message type is not supported."); +++ return response; +++ } +++ ++ try { ++ this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); ++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++index a815636b1..99a5db5ad 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++@@ -393,6 +393,8 @@ public class BrokerConfig extends BrokerIdentity { ++ */ ++ private boolean enableSingleTopicRegister = false; ++ +++ private boolean enableMixedMessageType = false; +++ ++ public long getMaxPopPollingSize() { ++ return maxPopPollingSize; ++ } ++@@ -1712,4 +1714,12 @@ public class BrokerConfig extends BrokerIdentity { ++ public void setEnableSingleTopicRegister(boolean enableSingleTopicRegister) { ++ this.enableSingleTopicRegister = enableSingleTopicRegister; ++ } +++ +++ public boolean isEnableMixedMessageType() { +++ return enableMixedMessageType; +++ } +++ +++ public void setEnableMixedMessageType(boolean enableMixedMessageType) { +++ this.enableMixedMessageType = enableMixedMessageType; +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From f534501855f8edbcb58f5b856973bf1027b5cf3a Mon Sep 17 00:00:00 2001 ++From: Steven ++Date: Fri, 11 Aug 2023 10:25:48 +0800 ++Subject: [PATCH 05/12] [Feature 7155] add errlog when cmd err (#7157) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Co-authored-by: 十真 ++--- ++ .../src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java ++index b00bad3c5..5a8a7cd54 100644 ++--- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java +++++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java ++@@ -52,6 +52,7 @@ public class ServerUtil { ++ System.exit(0); ++ } ++ } catch (ParseException e) { +++ System.err.println(e.getMessage()); ++ hf.printHelp(appName, options, true); ++ System.exit(1); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From db58f00c0fe0f129611d654291f2177de55dc9ff Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Fri, 11 Aug 2023 19:18:30 +0800 ++Subject: [PATCH 06/12] [ISSUE #7169] Change metadataThreadPoolQueueCapacity to ++ 100000 (#7170) ++ ++--- ++ .../main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 4f57a7052..39caaa0d9 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -165,7 +165,7 @@ public class ProxyConfig implements ConfigFile { ++ private int subscriptionGroupConfigCacheExpiredInSeconds = 20; ++ private int subscriptionGroupConfigCacheMaxNum = 20000; ++ private int metadataThreadPoolNums = 3; ++- private int metadataThreadPoolQueueCapacity = 1000; +++ private int metadataThreadPoolQueueCapacity = 100000; ++ ++ private int transactionHeartbeatThreadPoolNums = 20; ++ private int transactionHeartbeatThreadPoolQueueCapacity = 200; ++-- ++2.32.0.windows.2 ++ ++ ++From 1f04e68a2e331ab035b791280c5a91b60fe0c85f Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Sat, 12 Aug 2023 21:12:22 +0800 ++Subject: [PATCH 07/12] [ISSUE #7172] Unified Chinese for Name Server (#7173) ++ ++--- ++ docs/cn/concept.md | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/docs/cn/concept.md b/docs/cn/concept.md ++index cb2c863bd..3d67e9371 100644 ++--- a/docs/cn/concept.md +++++ b/docs/cn/concept.md ++@@ -17,7 +17,7 @@ RocketMQ主要由 Producer、Broker、Consumer 三部分组成，其中Producer ++ 消息中转角色，负责存储消息、转发消息。代理服务器在RocketMQ系统中负责接收从生产者发送来的消息并存储、同时为消费者的拉取请求作准备。代理服务器也存储消息相关的元数据，包括消费者组、消费进度偏移和主题和队列消息等。 ++ ++ ## 6 名字服务（Name Server） ++- 名称服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。 +++名字服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。 ++ ++ ## 7 拉取式消费（Pull Consumer） ++ Consumer消费的一种类型，应用通常主动调用Consumer的拉消息方法从Broker服务器拉消息、主动权由应用控制。一旦获取了批量消息，应用就会启动消费过程。 ++-- ++2.32.0.windows.2 ++ ++ ++From 25005060bbace477eeaaf4c0142cece5213efbbf Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Sun, 13 Aug 2023 20:52:17 +0800 ++Subject: [PATCH 08/12] [ISSUE #7176] Correct mismatched logs (#7177) ++ ++--- ++ .../org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java ++index 0055a1cc8..f7a95f0a6 100644 ++--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java ++@@ -522,7 +522,7 @@ public class RouteInfoManager { ++ this.lock.writeLock().unlock(); ++ } ++ } catch (Exception e) { ++- log.error("wipeWritePermOfBrokerByLock Exception", e); +++ log.error("addWritePermOfBrokerByLock Exception", e); ++ } ++ return 0; ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From ac411daa27117e9115a8fc5e2d5753085f009ed9 Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Tue, 15 Aug 2023 08:31:00 +0800 ++Subject: [PATCH 09/12] [ISSUE #7183] Correct mismatched commandDesc (#7184) ++ ++--- ++ .../tools/command/topic/RemappingStaticTopicSubCommand.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java ++index 849f680d0..2a08fdb5b 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java ++@@ -47,7 +47,7 @@ public class RemappingStaticTopicSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update or create static topic, which has fixed number of queues"; +++ return "Remapping static topic."; ++ } ++ ++ @Override ++-- ++2.32.0.windows.2 ++ ++ ++From 55e0cdb2af3ab75a6d892f919d60797f17a99fda Mon Sep 17 00:00:00 2001 ++From: redlsz ++Date: Tue, 15 Aug 2023 19:19:45 +0800 ++Subject: [PATCH 10/12] fix: IndexOutOfBoundsException when process pop ++ response (#7003) ++ ++--- ++ .../org/apache/rocketmq/client/impl/MQClientAPIImpl.java | 5 ++++- ++ .../rocketmq/proxy/service/message/LocalMessageService.java | 5 ++++- ++ .../rocketmq/remoting/protocol/header/ExtraInfoUtil.java | 4 ++++ ++ 3 files changed, 12 insertions(+), 2 deletions(-) ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 708a6acd1..5101ffc8e 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -1174,7 +1174,10 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET))); ++ continue; ++ } ++- key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId()); +++ // Value of POP_CK is used to determine whether it is a pop retry, +++ // cause topic could be rewritten by broker. +++ key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), +++ messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId()); ++ if (!sortMap.containsKey(key)) { ++ sortMap.put(key, new ArrayList<>(4)); ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++index 115c140ff..eb2c4d9ee 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++@@ -249,7 +249,10 @@ public class LocalMessageService implements MessageService { ++ // ++ Map> sortMap = new HashMap<>(16); ++ for (MessageExt messageExt : messageExtList) { ++- String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId()); +++ // Value of POP_CK is used to determine whether it is a pop retry, +++ // cause topic could be rewritten by broker. +++ String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), +++ messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId()); ++ if (!sortMap.containsKey(key)) { ++ sortMap.put(key, new ArrayList<>(4)); ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java ++index 9a5fa89ab..13094331e 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java ++@@ -282,6 +282,10 @@ public class ExtraInfoUtil { ++ return (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) ? RETRY_TOPIC : NORMAL_TOPIC) + "@" + key; ++ } ++ +++ public static String getStartOffsetInfoMapKey(String topic, String popCk, long key) { +++ return ((topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || popCk != null) ? RETRY_TOPIC : NORMAL_TOPIC) + "@" + key; +++ } +++ ++ public static String getQueueOffsetKeyValueKey(long queueId, long queueOffset) { ++ return QUEUE_OFFSET + queueId + "%" + queueOffset; ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From a9c0b43f7f6ce5acfc4f2f3069553071fa93dfee Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Wed, 16 Aug 2023 18:45:00 +0800 ++Subject: [PATCH 11/12] [ISSUE #7192] Correct typos (#7193) ++ ++--- ++ .../tools/command/consumer/ConsumerProgressSubCommand.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java ++index f51a24673..97125b854 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java ++@@ -54,7 +54,7 @@ public class ConsumerProgressSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query consumers's progress, speed"; +++ return "Query consumer's progress, speed."; ++ } ++ ++ @Override ++-- ++2.32.0.windows.2 ++ ++ ++From 5a3de926b816db5a121c1d788430072a3bc942ae Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Wed, 16 Aug 2023 20:52:53 +0800 ++Subject: [PATCH 12/12] Optimize updateSubscription check exist loop (#7190) ++ ++--- ++ .../broker/client/ConsumerGroupInfo.java | 17 ++++++----------- ++ 1 file changed, 6 insertions(+), 11 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java ++index 867b9c720..1ea58c125 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java ++@@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.client; ++ ++ import io.netty.channel.Channel; ++ import java.util.ArrayList; +++import java.util.HashSet; ++ import java.util.Iterator; ++ import java.util.List; ++ import java.util.Map.Entry; ++@@ -172,7 +173,7 @@ public class ConsumerGroupInfo { ++ */ ++ public boolean updateSubscription(final Set subList) { ++ boolean updated = false; ++- +++ Set topicSet = new HashSet<>(); ++ for (SubscriptionData sub : subList) { ++ SubscriptionData old = this.subscriptionTable.get(sub.getTopic()); ++ if (old == null) { ++@@ -194,22 +195,16 @@ public class ConsumerGroupInfo { ++ ++ this.subscriptionTable.put(sub.getTopic(), sub); ++ } +++ // Add all new topics to the HashSet +++ topicSet.add(sub.getTopic()); ++ } ++ ++ Iterator> it = this.subscriptionTable.entrySet().iterator(); ++ while (it.hasNext()) { ++ Entry next = it.next(); ++ String oldTopic = next.getKey(); ++- ++- boolean exist = false; ++- for (SubscriptionData sub : subList) { ++- if (sub.getTopic().equals(oldTopic)) { ++- exist = true; ++- break; ++- } ++- } ++- ++- if (!exist) { +++ // Check HashSet with O(1) time complexity +++ if (!topicSet.contains(oldTopic)) { ++ log.warn("subscription changed, group: {} remove topic {} {}", ++ this.groupName, ++ oldTopic, ++-- ++2.32.0.windows.2 ++ +diff --git a/patch011-backport-optimize-config b/patch011-backport-optimize-config +new file mode 100644 +index 000000000..fc8e2eceb +--- /dev/null ++++ b/patch011-backport-optimize-config +@@ -0,0 +1,1390 @@ ++From 50d1050437ed8748f86ee50261b50a1e1f63162e Mon Sep 17 00:00:00 2001 ++From: Jixiang Jin ++Date: Wed, 16 Aug 2023 21:15:00 +0800 ++Subject: [PATCH 1/7] To config the cardinalityLimit for openTelemetry metrics ++ exporting and fix logging config for metrics (#7196) ++ ++--- ++ WORKSPACE | 14 +++--- ++ .../broker/metrics/BrokerMetricsManager.java | 47 ++++++++++++++----- ++ .../broker/metrics/PopMetricsManager.java | 11 +++-- ++ .../src/main/resources/rmq.broker.logback.xml | 17 ++++--- ++ .../apache/rocketmq/common/BrokerConfig.java | 9 ++++ ++ .../metrics/ControllerMetricsManager.java | 6 +-- ++ pom.xml | 4 +- ++ .../metrics/RemotingMetricsManager.java | 10 ++-- ++ .../rocketmq/store/DefaultMessageStore.java | 24 +++++----- ++ .../apache/rocketmq/store/MessageStore.java | 6 +-- ++ .../metrics/DefaultStoreMetricsManager.java | 4 +- ++ .../plugin/AbstractPluginMessageStore.java | 6 +-- ++ .../tieredstore/TieredMessageStore.java | 6 +-- ++ .../metrics/TieredStoreMetricsManager.java | 23 +++++---- ++ 14 files changed, 110 insertions(+), 77 deletions(-) ++ ++diff --git a/WORKSPACE b/WORKSPACE ++index a8a0aafe9..3126f2d1d 100644 ++--- a/WORKSPACE +++++ b/WORKSPACE ++@@ -88,14 +88,14 @@ maven_install( ++ "io.grpc:grpc-api:1.47.0", ++ "io.grpc:grpc-testing:1.47.0", ++ "org.springframework:spring-core:5.3.26", ++- "io.opentelemetry:opentelemetry-exporter-otlp:1.19.0", ++- "io.opentelemetry:opentelemetry-exporter-prometheus:1.19.0-alpha", ++- "io.opentelemetry:opentelemetry-exporter-logging:1.19.0", ++- "io.opentelemetry:opentelemetry-sdk:1.19.0", +++ "io.opentelemetry:opentelemetry-exporter-otlp:1.29.0", +++ "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha", +++ "io.opentelemetry:opentelemetry-exporter-logging:1.29.0", +++ "io.opentelemetry:opentelemetry-sdk:1.29.0", ++ "com.squareup.okio:okio-jvm:3.0.0", ++- "io.opentelemetry:opentelemetry-api:1.19.0", ++- "io.opentelemetry:opentelemetry-sdk-metrics:1.19.0", ++- "io.opentelemetry:opentelemetry-sdk-common:1.19.0", +++ "io.opentelemetry:opentelemetry-api:1.29.0", +++ "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0", +++ "io.opentelemetry:opentelemetry-sdk-common:1.29.0", ++ "io.github.aliyunmq:rocketmq-slf4j-api:1.0.0", ++ "io.github.aliyunmq:rocketmq-logback-classic:1.0.0", ++ "org.slf4j:jul-to-slf4j:2.0.6", ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++index f0b76107e..6af5afc14 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++@@ -34,8 +34,10 @@ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.SdkMeterProvider; ++ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; ++ import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import io.opentelemetry.sdk.metrics.data.AggregationTemporality; ++ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +++import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; ++ import io.opentelemetry.sdk.resources.Resource; ++ import java.util.ArrayList; ++ import java.util.Arrays; ++@@ -361,22 +363,45 @@ public class BrokerMetricsManager { ++ .setType(InstrumentType.HISTOGRAM) ++ .setName(HISTOGRAM_MESSAGE_SIZE) ++ .build(); ++- View messageSizeView = View.builder() ++- .setAggregation(Aggregation.explicitBucketHistogram(messageSizeBuckets)) ++- .build(); ++- providerBuilder.registerView(messageSizeSelector, messageSizeView); ++- ++- for (Pair selectorViewPair : RemotingMetricsManager.getMetricsView()) { ++- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2()); +++ ViewBuilder messageSizeViewBuilder = View.builder() +++ .setAggregation(Aggregation.explicitBucketHistogram(messageSizeBuckets)); +++ // To config the cardinalityLimit for openTelemetry metrics exporting. +++ SdkMeterProviderUtil.setCardinalityLimit(messageSizeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(messageSizeSelector, messageSizeViewBuilder.build()); +++ +++ for (Pair selectorViewPair : RemotingMetricsManager.getMetricsView()) { +++ ViewBuilder viewBuilder = selectorViewPair.getObject2(); +++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build()); ++ } ++ ++- for (Pair selectorViewPair : messageStore.getMetricsView()) { ++- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2()); +++ for (Pair selectorViewPair : messageStore.getMetricsView()) { +++ ViewBuilder viewBuilder = selectorViewPair.getObject2(); +++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build()); ++ } ++ ++- for (Pair selectorViewPair : PopMetricsManager.getMetricsView()) { ++- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2()); +++ for (Pair selectorViewPair : PopMetricsManager.getMetricsView()) { +++ ViewBuilder viewBuilder = selectorViewPair.getObject2(); +++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build()); ++ } +++ +++ // default view builder for all counter. +++ InstrumentSelector defaultCounterSelector = InstrumentSelector.builder() +++ .setType(InstrumentType.COUNTER) +++ .build(); +++ ViewBuilder defaultCounterViewBuilder = View.builder().setDescription("default view for counter."); +++ SdkMeterProviderUtil.setCardinalityLimit(defaultCounterViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(defaultCounterSelector, defaultCounterViewBuilder.build()); +++ +++ //default view builder for all observable gauge. +++ InstrumentSelector defaultGaugeSelector = InstrumentSelector.builder() +++ .setType(InstrumentType.OBSERVABLE_GAUGE) +++ .build(); +++ ViewBuilder defaultGaugeViewBuilder = View.builder().setDescription("default view for gauge."); +++ SdkMeterProviderUtil.setCardinalityLimit(defaultGaugeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(defaultGaugeSelector, defaultGaugeViewBuilder.build()); ++ } ++ ++ private void initStatsMetrics() { ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java ++index 463371d7e..2de220da1 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java ++@@ -27,6 +27,7 @@ import io.opentelemetry.sdk.metrics.Aggregation; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.time.Duration; ++ import java.util.Arrays; ++ import java.util.List; ++@@ -63,7 +64,7 @@ public class PopMetricsManager { ++ private static LongCounter popReviveGetTotal = new NopLongCounter(); ++ private static LongCounter popReviveRetryMessageTotal = new NopLongCounter(); ++ ++- public static List> getMetricsView() { +++ public static List> getMetricsView() { ++ List rpcCostTimeBuckets = Arrays.asList( ++ (double) Duration.ofMillis(1).toMillis(), ++ (double) Duration.ofMillis(10).toMillis(), ++@@ -76,10 +77,10 @@ public class PopMetricsManager { ++ .setType(InstrumentType.HISTOGRAM) ++ .setName(HISTOGRAM_POP_BUFFER_SCAN_TIME_CONSUME) ++ .build(); ++- View popBufferScanTimeConsumeView = View.builder() ++- .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)) ++- .build(); ++- return Lists.newArrayList(new Pair<>(popBufferScanTimeConsumeSelector, popBufferScanTimeConsumeView)); +++ ViewBuilder popBufferScanTimeConsumeViewBuilder = View.builder() +++ .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)); +++ +++ return Lists.newArrayList(new Pair<>(popBufferScanTimeConsumeSelector, popBufferScanTimeConsumeViewBuilder)); ++ } ++ ++ public static void initMetrics(Meter meter, BrokerController brokerController, ++diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml ++index 7d49f6664..3c51e59d4 100644 ++--- a/broker/src/main/resources/rmq.broker.logback.xml +++++ b/broker/src/main/resources/rmq.broker.logback.xml ++@@ -559,27 +559,27 @@ ++ ++ ++ ++- +++ ++ ++ brokerContainerLogDir ++ ${file.separator} ++ ++ ++- ++ ++- ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_metric.log +++ ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_metrics.log ++ ++ true ++ ++ ++- ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_metric.%i.log.gz +++ ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_metrics.%i.log.gz ++ ++ 1 ++- 10 +++ 3 ++ ++ ++- 500MB +++ 512MB ++ ++ ++ %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n ++@@ -588,6 +588,9 @@ ++ ++ ++ +++ +++ +++ ++ ++ ++ ++@@ -670,7 +673,7 @@ ++ ++ ++ ++- +++ ++ ++ ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++index 99a5db5ad..45d26b29c 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++@@ -350,6 +350,7 @@ public class BrokerConfig extends BrokerIdentity { ++ ++ private MetricsExporterType metricsExporterType = MetricsExporterType.DISABLE; ++ +++ private int metricsOtelCardinalityLimit = 50 * 1000; ++ private String metricsGrpcExporterTarget = ""; ++ private String metricsGrpcExporterHeader = ""; ++ private long metricGrpcExporterTimeOutInMills = 3 * 1000; ++@@ -1531,6 +1532,14 @@ public class BrokerConfig extends BrokerIdentity { ++ this.metricsExporterType = MetricsExporterType.valueOf(metricsExporterType); ++ } ++ +++ public int getMetricsOtelCardinalityLimit() { +++ return metricsOtelCardinalityLimit; +++ } +++ +++ public void setMetricsOtelCardinalityLimit(int metricsOtelCardinalityLimit) { +++ this.metricsOtelCardinalityLimit = metricsOtelCardinalityLimit; +++ } +++ ++ public String getMetricsGrpcExporterTarget() { ++ return metricsGrpcExporterTarget; ++ } ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++index 9b30a3b43..650740bcc 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++@@ -203,7 +203,7 @@ public class ControllerMetricsManager { ++ 10 * s ++ ); ++ ++- View latecyView = View.builder() +++ View latencyView = View.builder() ++ .setAggregation(Aggregation.explicitBucketHistogram(latencyBuckets)) ++ .build(); ++ ++@@ -217,8 +217,8 @@ public class ControllerMetricsManager { ++ .setName(HISTOGRAM_DLEDGER_OP_LATENCY) ++ .build(); ++ ++- providerBuilder.registerView(requestLatencySelector, latecyView); ++- providerBuilder.registerView(dLedgerOpLatencySelector, latecyView); +++ providerBuilder.registerView(requestLatencySelector, latencyView); +++ providerBuilder.registerView(dLedgerOpLatencySelector, latencyView); ++ } ++ ++ private void initMetric(Meter meter) { ++diff --git a/pom.xml b/pom.xml ++index 3a08d75f2..9f0b3eb96 100644 ++--- a/pom.xml +++++ b/pom.xml ++@@ -133,8 +133,8 @@ ++ 2.9.3 ++ 5.3.27 ++ 3.0.0 ++- 1.26.0 ++- 1.26.0-alpha +++ 1.29.0 +++ 1.29.0-alpha ++ 2.0.6 ++ 2.20.29 ++ 1.0.3 ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java b/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java ++index 34136f94f..2e0d70856 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java ++@@ -26,6 +26,7 @@ import io.opentelemetry.sdk.metrics.Aggregation; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.time.Duration; ++ import java.util.Arrays; ++ import java.util.List; ++@@ -61,7 +62,7 @@ public class RemotingMetricsManager { ++ .build(); ++ } ++ ++- public static List> getMetricsView() { +++ public static List> getMetricsView() { ++ List rpcCostTimeBuckets = Arrays.asList( ++ (double) Duration.ofMillis(1).toMillis(), ++ (double) Duration.ofMillis(3).toMillis(), ++@@ -77,10 +78,9 @@ public class RemotingMetricsManager { ++ .setType(InstrumentType.HISTOGRAM) ++ .setName(HISTOGRAM_RPC_LATENCY) ++ .build(); ++- View view = View.builder() ++- .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)) ++- .build(); ++- return Lists.newArrayList(new Pair<>(selector, view)); +++ ViewBuilder viewBuilder = View.builder() +++ .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)); +++ return Lists.newArrayList(new Pair<>(selector, viewBuilder)); ++ } ++ ++ public static String getWriteAndFlushResult(Future future) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++index 25e4a166f..6115ead59 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++@@ -22,7 +22,7 @@ import io.openmessaging.storage.dledger.entry.DLedgerEntry; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.io.File; ++ import java.io.IOException; ++ import java.io.RandomAccessFile; ++@@ -42,23 +42,24 @@ import java.util.Map; ++ import java.util.Objects; ++ import java.util.Optional; ++ import java.util.Set; ++-import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.TimeUnit; ++-import java.util.concurrent.TimeoutException; ++-import java.util.concurrent.ExecutionException; ++-import java.util.concurrent.Executors; ++-import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.ThreadPoolExecutor; ++-import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.CompletableFuture; ++-import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentLinkedQueue; +++import java.util.concurrent.ConcurrentMap; +++import java.util.concurrent.ExecutionException; +++import java.util.concurrent.ExecutorService; +++import java.util.concurrent.Executors; +++import java.util.concurrent.LinkedBlockingQueue; +++import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ThreadPoolExecutor; +++import java.util.concurrent.TimeUnit; +++import java.util.concurrent.TimeoutException; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.concurrent.atomic.AtomicLong; ++ import java.util.function.Supplier; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.AbstractBrokerRunnable; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.BrokerConfig; ++ import org.apache.rocketmq.common.BrokerIdentity; ++ import org.apache.rocketmq.common.MixAll; ++@@ -82,7 +83,6 @@ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.common.utils.CleanupPolicyUtils; ++ import org.apache.rocketmq.common.utils.QueueTypeUtils; ++ import org.apache.rocketmq.common.utils.ServiceProvider; ++-import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; ++@@ -3268,7 +3268,7 @@ public class DefaultMessageStore implements MessageStore { ++ } ++ ++ @Override ++- public List> getMetricsView() { +++ public List> getMetricsView() { ++ return DefaultStoreMetricsManager.getMetricsView(); ++ } ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java ++index 31bbb907f..989cbbe31 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java ++@@ -19,8 +19,7 @@ package org.apache.rocketmq.store; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; ++- +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.nio.ByteBuffer; ++ import java.util.HashMap; ++ import java.util.LinkedList; ++@@ -28,7 +27,6 @@ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.function.Supplier; ++- ++ import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.SystemClock; ++@@ -964,7 +962,7 @@ public interface MessageStore { ++ * ++ * @return List of metrics selector and view pair ++ */ ++- List> getMetricsView(); +++ List> getMetricsView(); ++ ++ /** ++ * Init store metrics ++diff --git a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java ++index ff87f6369..45a6bbc68 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java +++++ b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java ++@@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongCounter; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.api.metrics.ObservableLongGauge; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.io.File; ++ import java.util.List; ++ import java.util.function.Supplier; ++@@ -69,7 +69,7 @@ public class DefaultStoreMetricsManager { ++ public static LongCounter timerDequeueTotal = new NopLongCounter(); ++ public static LongCounter timerEnqueueTotal = new NopLongCounter(); ++ ++- public static List> getMetricsView() { +++ public static List> getMetricsView() { ++ return Lists.newArrayList(); ++ } ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java ++index 25e947512..ab9fc6da7 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java ++@@ -20,8 +20,7 @@ package org.apache.rocketmq.store.plugin; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; ++- +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.nio.ByteBuffer; ++ import java.util.HashMap; ++ import java.util.LinkedList; ++@@ -29,7 +28,6 @@ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.function.Supplier; ++- ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.SystemClock; ++ import org.apache.rocketmq.common.message.MessageExt; ++@@ -643,7 +641,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { ++ } ++ ++ @Override ++- public List> getMetricsView() { +++ public List> getMetricsView() { ++ return next.getMetricsView(); ++ } ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index ced1fb818..5240ac8e9 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -21,7 +21,7 @@ import io.opentelemetry.api.common.Attributes; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++@@ -352,8 +352,8 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ } ++ ++ @Override ++- public List> getMetricsView() { ++- List> res = super.getMetricsView(); +++ public List> getMetricsView() { +++ List> res = super.getMetricsView(); ++ res.addAll(TieredStoreMetricsManager.getMetricsView()); ++ return res; ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java ++index 3ca0fb614..d8a07f0a7 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java ++@@ -27,6 +27,7 @@ import io.opentelemetry.sdk.metrics.Aggregation; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.util.ArrayList; ++ import java.util.Arrays; ++ import java.util.HashMap; ++@@ -101,8 +102,8 @@ public class TieredStoreMetricsManager { ++ public static ObservableLongGauge storageSize = new NopObservableLongGauge(); ++ public static ObservableLongGauge storageMessageReserveTime = new NopObservableLongGauge(); ++ ++- public static List> getMetricsView() { ++- ArrayList> res = new ArrayList<>(); +++ public static List> getMetricsView() { +++ ArrayList> res = new ArrayList<>(); ++ ++ InstrumentSelector providerRpcLatencySelector = InstrumentSelector.builder() ++ .setType(InstrumentType.HISTOGRAM) ++@@ -114,10 +115,9 @@ public class TieredStoreMetricsManager { ++ .setName(HISTOGRAM_API_LATENCY) ++ .build(); ++ ++- View rpcLatencyView = View.builder() +++ ViewBuilder rpcLatencyViewBuilder = View.builder() ++ .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d, 3d, 5d, 7d, 10d, 100d, 200d, 400d, 600d, 800d, 1d * 1000, 1d * 1500, 1d * 3000))) ++- .setDescription("tiered_store_rpc_latency_view") ++- .build(); +++ .setDescription("tiered_store_rpc_latency_view"); ++ ++ InstrumentSelector uploadBufferSizeSelector = InstrumentSelector.builder() ++ .setType(InstrumentType.HISTOGRAM) ++@@ -129,15 +129,14 @@ public class TieredStoreMetricsManager { ++ .setName(HISTOGRAM_DOWNLOAD_BYTES) ++ .build(); ++ ++- View bufferSizeView = View.builder() +++ ViewBuilder bufferSizeViewBuilder = View.builder() ++ .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d * TieredStoreUtil.KB, 10d * TieredStoreUtil.KB, 100d * TieredStoreUtil.KB, 1d * TieredStoreUtil.MB, 10d * TieredStoreUtil.MB, 32d * TieredStoreUtil.MB, 50d * TieredStoreUtil.MB, 100d * TieredStoreUtil.MB))) ++- .setDescription("tiered_store_buffer_size_view") ++- .build(); +++ .setDescription("tiered_store_buffer_size_view"); ++ ++- res.add(new Pair<>(rpcLatencySelector, rpcLatencyView)); ++- res.add(new Pair<>(providerRpcLatencySelector, rpcLatencyView)); ++- res.add(new Pair<>(uploadBufferSizeSelector, bufferSizeView)); ++- res.add(new Pair<>(downloadBufferSizeSelector, bufferSizeView)); +++ res.add(new Pair<>(rpcLatencySelector, rpcLatencyViewBuilder)); +++ res.add(new Pair<>(providerRpcLatencySelector, rpcLatencyViewBuilder)); +++ res.add(new Pair<>(uploadBufferSizeSelector, bufferSizeViewBuilder)); +++ res.add(new Pair<>(downloadBufferSizeSelector, bufferSizeViewBuilder)); ++ return res; ++ } ++ ++-- ++2.32.0.windows.2 ++ ++ ++From a4bcc2a74d8bec9c9d34565536e87df06e0b11c1 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Thu, 17 Aug 2023 13:53:48 +0800 ++Subject: [PATCH 2/7] [ISSUE #7178] refresh metadata after broker startup ++ ++Signed-off-by: Ziy1-Tan ++--- ++ .../rocketmq/broker/BrokerController.java | 24 +++++++++---------- ++ 1 file changed, 12 insertions(+), 12 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 30b1d2299..13f9d002b 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -663,7 +663,7 @@ public class BrokerController { ++ BrokerController.this.getSlaveSynchronize().syncAll(); ++ lastSyncTimeMs = System.currentTimeMillis(); ++ } ++- +++ ++ //timer checkpoint, latency-sensitive, so sync it more frequently ++ if (messageStoreConfig.isTimerWheelEnable()) { ++ BrokerController.this.getSlaveSynchronize().syncTimerCheckPoint(); ++@@ -698,17 +698,6 @@ public class BrokerController { ++ ++ initializeBrokerScheduledTasks(); ++ ++- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { ++- @Override ++- public void run() { ++- try { ++- BrokerController.this.brokerOuterAPI.refreshMetadata(); ++- } catch (Exception e) { ++- LOG.error("ScheduledTask refresh metadata exception", e); ++- } ++- } ++- }, 10, 5, TimeUnit.SECONDS); ++- ++ if (this.brokerConfig.getNamesrvAddr() != null) { ++ this.updateNamesrvAddr(); ++ LOG.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr()); ++@@ -1682,6 +1671,17 @@ public class BrokerController { ++ if (brokerConfig.isSkipPreOnline()) { ++ startServiceWithoutCondition(); ++ } +++ +++ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { +++ @Override +++ public void run() { +++ try { +++ BrokerController.this.brokerOuterAPI.refreshMetadata(); +++ } catch (Exception e) { +++ LOG.error("ScheduledTask refresh metadata exception", e); +++ } +++ } +++ }, 10, 5, TimeUnit.SECONDS); ++ } ++ ++ protected void scheduleSendHeartbeat() { ++-- ++2.32.0.windows.2 ++ ++ ++From 3df1b9232af99944cb3d4d4d2d00c5a85cd3b57d Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Thu, 17 Aug 2023 13:59:04 +0800 ++Subject: [PATCH 3/7] [ISSUE #7201] Remove the DefaultMessageStore.class ++ dependency in TransientStorePool ++ ++Co-authored-by: guyinyou ++--- ++ .../rocketmq/store/AllocateMappedFileService.java | 6 +++--- ++ .../apache/rocketmq/store/DefaultMessageStore.java | 7 +++++-- ++ .../apache/rocketmq/store/TransientStorePool.java | 13 ++++--------- ++ 3 files changed, 12 insertions(+), 14 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java ++index dca7d5325..c8420fea1 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java +++++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java ++@@ -55,7 +55,7 @@ public class AllocateMappedFileService extends ServiceThread { ++ if (this.messageStore.isTransientStorePoolEnable()) { ++ if (this.messageStore.getMessageStoreConfig().isFastFailIfNoBufferInStorePool() ++ && BrokerRole.SLAVE != this.messageStore.getMessageStoreConfig().getBrokerRole()) { //if broker is slave, don't fast fail even no buffer in pool ++- canSubmitRequests = this.messageStore.getTransientStorePool().availableBufferNums() - this.requestQueue.size(); +++ canSubmitRequests = this.messageStore.remainTransientStoreBufferNumbs() - this.requestQueue.size(); ++ } ++ } ++ ++@@ -65,7 +65,7 @@ public class AllocateMappedFileService extends ServiceThread { ++ if (nextPutOK) { ++ if (canSubmitRequests <= 0) { ++ log.warn("[NOTIFYME]TransientStorePool is not enough, so create mapped file error, " + ++- "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums()); +++ "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs()); ++ this.requestTable.remove(nextFilePath); ++ return null; ++ } ++@@ -81,7 +81,7 @@ public class AllocateMappedFileService extends ServiceThread { ++ if (nextNextPutOK) { ++ if (canSubmitRequests <= 0) { ++ log.warn("[NOTIFYME]TransientStorePool is not enough, so skip preallocate mapped file, " + ++- "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums()); +++ "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs()); ++ this.requestTable.remove(nextNextFilePath); ++ } else { ++ boolean offerOK = this.requestQueue.offer(nextNextReq); ++diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++index 6115ead59..f2a54ddf6 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++@@ -250,7 +250,7 @@ public class DefaultMessageStore implements MessageStore { ++ this.reputMessageService = new ConcurrentReputMessageService(); ++ } ++ ++- this.transientStorePool = new TransientStorePool(this); +++ this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog()); ++ ++ this.scheduledExecutorService = ++ Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); ++@@ -1983,7 +1983,10 @@ public class DefaultMessageStore implements MessageStore { ++ } ++ ++ public int remainTransientStoreBufferNumbs() { ++- return this.transientStorePool.availableBufferNums(); +++ if (this.isTransientStorePoolEnable()) { +++ return this.transientStorePool.availableBufferNums(); +++ } +++ return Integer.MAX_VALUE; ++ } ++ ++ @Override ++diff --git a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java ++index 8c1a5338b..0d42ee69e 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java +++++ b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java ++@@ -33,13 +33,11 @@ public class TransientStorePool { ++ private final int poolSize; ++ private final int fileSize; ++ private final Deque availableBuffers; ++- private final DefaultMessageStore messageStore; ++ private volatile boolean isRealCommit = true; ++ ++- public TransientStorePool(final DefaultMessageStore messageStore) { ++- this.messageStore = messageStore; ++- this.poolSize = messageStore.getMessageStoreConfig().getTransientStorePoolSize(); ++- this.fileSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); +++ public TransientStorePool(final int poolSize, final int fileSize) { +++ this.poolSize = poolSize; +++ this.fileSize = fileSize; ++ this.availableBuffers = new ConcurrentLinkedDeque<>(); ++ } ++ ++@@ -81,10 +79,7 @@ public class TransientStorePool { ++ } ++ ++ public int availableBufferNums() { ++- if (messageStore.isTransientStorePoolEnable()) { ++- return availableBuffers.size(); ++- } ++- return Integer.MAX_VALUE; +++ return availableBuffers.size(); ++ } ++ ++ public boolean isRealCommit() { ++-- ++2.32.0.windows.2 ++ ++ ++From 2b93e1e32fd458d9df2091e89ea259ddd4d54061 Mon Sep 17 00:00:00 2001 ++From: iamgd67 ++Date: Thu, 17 Aug 2023 15:31:14 +0800 ++Subject: [PATCH 4/7] Update mqbroker to use runbroker.sh instead of ++ runserver.sh when use --enable-proxy (#7150) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Update mqbroker to use runbroker.sh instead of runserver.sh when enabling `--enable-proxy` ++this allow JVM `heap` and `gc` configuration using broker's settings instead of other common serverices'(proxy,namenode, etc). ++our main purpose, like the filename `mqbroker` suggest, is to start broker (which embeds a proxy), so use broker's config is reasonable ++ ++chinese version ++mqbroker的--enable-proxy选项是启动内嵌了proxy的broker，而不是内嵌broker的proxy，而且broker的工作量和重要程度大于proxy，所以使用broker的gc和heap配置更合适 ++--- ++ distribution/bin/mqbroker | 4 ++-- ++ 1 file changed, 2 insertions(+), 2 deletions(-) ++ ++diff --git a/distribution/bin/mqbroker b/distribution/bin/mqbroker ++index 3758ed597..35eb93c44 100644 ++--- a/distribution/bin/mqbroker +++++ b/distribution/bin/mqbroker ++@@ -68,11 +68,11 @@ if [ "$enable_proxy" = true ]; then ++ if [ "$broker_config" != "" ]; then ++ args_for_proxy=${args_for_proxy}" -bc "${broker_config} ++ fi ++- sh ${ROCKETMQ_HOME}/bin/runserver.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy} +++ sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy} ++ else ++ args_for_broker=$other_args ++ if [ "$broker_config" != "" ]; then ++ args_for_broker=${args_for_broker}" -c "${broker_config} ++ fi ++ sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.broker.logback.xml org.apache.rocketmq.broker.BrokerStartup ${args_for_broker} ++-fi ++\ No newline at end of file +++fi ++-- ++2.32.0.windows.2 ++ ++ ++From 05e7cde610255ed9410fffb0f153efe7c2c8a326 Mon Sep 17 00:00:00 2001 ++From: yao-wenbin ++Date: Fri, 18 Aug 2023 09:49:59 +0800 ++Subject: [PATCH 5/7] [ISSUE #7042] maven-compile job failed, Because TlsTest's ++ serverRejectsSSLClient test case will throw TooLongFrameException (#7179) ++ ++--- ++ .../remoting/netty/NettyRemotingServer.java | 2 +- ++ .../java/org/apache/rocketmq/remoting/TlsTest.java | 14 ++++++++++++-- ++ 2 files changed, 13 insertions(+), 3 deletions(-) ++ ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++index 90e358ce3..17f138f86 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++@@ -502,7 +502,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ case DISABLED: ++ ctx.close(); ++ log.warn("Clients intend to establish an SSL connection while this server is running in SSL disabled mode"); ++- break; +++ throw new UnsupportedOperationException("The NettyRemotingServer in SSL disabled mode doesn't support ssl client"); ++ case PERMISSIVE: ++ case ENFORCING: ++ if (null != sslContext) { ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java ++index de7edbbfb..a4890d73d 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java ++@@ -144,8 +144,13 @@ public class TlsTest { ++ tlsClientKeyPath = ""; ++ tlsClientCertPath = ""; ++ clientConfig.setUseTLS(false); ++- } else if ("serverRejectsSSLClient".equals(name.getMethodName())) { +++ } else if ("disabledServerRejectsSSLClient".equals(name.getMethodName())) { ++ tlsMode = TlsMode.DISABLED; +++ } else if ("disabledServerAcceptUnAuthClient".equals(name.getMethodName())) { +++ tlsMode = TlsMode.DISABLED; +++ tlsClientKeyPath = ""; +++ tlsClientCertPath = ""; +++ clientConfig.setUseTLS(false); ++ } else if ("reloadSslContextForServer".equals(name.getMethodName())) { ++ tlsClientAuthServer = false; ++ tlsServerNeedClientAuth = "none"; ++@@ -211,7 +216,7 @@ public class TlsTest { ++ } ++ ++ @Test ++- public void serverRejectsSSLClient() throws Exception { +++ public void disabledServerRejectsSSLClient() throws Exception { ++ try { ++ RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 5); ++ failBecauseExceptionWasNotThrown(RemotingSendRequestException.class); ++@@ -219,6 +224,11 @@ public class TlsTest { ++ } ++ } ++ +++ @Test +++ public void disabledServerAcceptUnAuthClient() throws Exception { +++ requestThenAssertResponse(); +++ } +++ ++ /** ++ * Tests that a server configured to require client authentication refuses to accept connections ++ * from a client that has an untrusted certificate. ++-- ++2.32.0.windows.2 ++ ++ ++From 72d796f2b20b3ec6aebca8c004d9275d7c749a95 Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Fri, 18 Aug 2023 11:55:39 +0800 ++Subject: [PATCH 6/7] [ISSUE #7205] support batch ack for pop orderly (#7206) ++ ++--- ++ .../broker/processor/AckMessageProcessor.java | 99 ++++++----- ++ .../rocketmq/client/impl/MQClientAPIImpl.java | 91 ++++++++-- ++ .../test/client/rmq/RMQPopClient.java | 22 +++ ++ .../client/consumer/pop/BasePopNormally.java | 6 + ++ .../test/client/consumer/pop/BatchAckIT.java | 159 ++++++++++++++++++ ++ 5 files changed, 322 insertions(+), 55 deletions(-) ++ create mode 100644 test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java ++index 687811409..244b459d6 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.processor; ++ import com.alibaba.fastjson.JSON; ++ import io.netty.channel.Channel; ++ import io.netty.channel.ChannelHandlerContext; +++import java.util.BitSet; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.broker.metrics.PopMetricsManager; ++ import org.apache.rocketmq.common.KeyBuilder; ++@@ -186,46 +187,7 @@ public class AckMessageProcessor implements NettyRequestProcessor { ++ invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfo); ++ ++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) { ++- // order ++- String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId; ++- long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); ++- if (ackOffset < oldOffset) { ++- return; ++- } ++- while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) { ++- } ++- try { ++- oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); ++- if (ackOffset < oldOffset) { ++- return; ++- } ++- long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext( ++- topic, consumeGroup, ++- qId, ackOffset, ++- popTime); ++- if (nextOffset > -1) { ++- if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset( ++- topic, consumeGroup, qId)) { ++- this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), ++- consumeGroup, topic, qId, nextOffset); ++- } ++- if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, ++- consumeGroup, qId, invisibleTime)) { ++- this.brokerController.getPopMessageProcessor().notifyMessageArriving( ++- topic, consumeGroup, qId); ++- } ++- } else if (nextOffset == -1) { ++- String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", ++- lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress()); ++- POP_LOGGER.warn(errorInfo); ++- response.setCode(ResponseCode.MESSAGE_ILLEGAL); ++- response.setRemark(errorInfo); ++- return; ++- } ++- } finally { ++- this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey); ++- } ++- brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount); +++ ackOrderly(topic, consumeGroup, qId, ackOffset, popTime, invisibleTime, channel, response); ++ return; ++ } ++ ++@@ -250,17 +212,22 @@ public class AckMessageProcessor implements NettyRequestProcessor { ++ } ++ ++ BatchAckMsg batchAckMsg = new BatchAckMsg(); ++- for (int i = 0; batchAck.getBitSet() != null && i < batchAck.getBitSet().length(); i++) { ++- if (!batchAck.getBitSet().get(i)) { ++- continue; +++ BitSet bitSet = batchAck.getBitSet(); +++ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { +++ if (i == Integer.MAX_VALUE) { +++ break; ++ } ++ long offset = startOffset + i; ++ if (offset < minOffset || offset > maxOffset) { ++ continue; ++ } ++- batchAckMsg.getAckOffsetList().add(offset); +++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) { +++ ackOrderly(topic, consumeGroup, qId, offset, popTime, invisibleTime, channel, response); +++ } else { +++ batchAckMsg.getAckOffsetList().add(offset); +++ } ++ } ++- if (batchAckMsg.getAckOffsetList().isEmpty()) { +++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE || batchAckMsg.getAckOffsetList().isEmpty()) { ++ return; ++ } ++ ++@@ -311,4 +278,46 @@ public class AckMessageProcessor implements NettyRequestProcessor { ++ PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus()); ++ brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount); ++ } +++ +++ protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime, long invisibleTime, Channel channel, RemotingCommand response) { +++ String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId; +++ long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); +++ if (ackOffset < oldOffset) { +++ return; +++ } +++ while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) { +++ } +++ try { +++ oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); +++ if (ackOffset < oldOffset) { +++ return; +++ } +++ long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext( +++ topic, consumeGroup, +++ qId, ackOffset, +++ popTime); +++ if (nextOffset > -1) { +++ if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset( +++ topic, consumeGroup, qId)) { +++ this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), +++ consumeGroup, topic, qId, nextOffset); +++ } +++ if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, +++ consumeGroup, qId, invisibleTime)) { +++ this.brokerController.getPopMessageProcessor().notifyMessageArriving( +++ topic, consumeGroup, qId); +++ } +++ } else if (nextOffset == -1) { +++ String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", +++ lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress()); +++ POP_LOGGER.warn(errorInfo); +++ response.setCode(ResponseCode.MESSAGE_ILLEGAL); +++ response.setRemark(errorInfo); +++ return; +++ } +++ } finally { +++ this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey); +++ } +++ brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, 1); +++ } ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 5101ffc8e..213c26fd6 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -21,6 +21,7 @@ import java.io.UnsupportedEncodingException; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; ++ import java.util.Arrays; +++import java.util.BitSet; ++ import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.Iterator; ++@@ -54,6 +55,7 @@ import org.apache.rocketmq.client.producer.SendCallback; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.client.producer.SendStatus; ++ import org.apache.rocketmq.common.AclConfig; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.MQVersion; ++ import org.apache.rocketmq.common.MixAll; ++ import org.apache.rocketmq.common.Pair; ++@@ -76,7 +78,8 @@ import org.apache.rocketmq.common.namesrv.NameServerUpdateCallback; ++ import org.apache.rocketmq.common.namesrv.TopAddressing; ++ import org.apache.rocketmq.common.sysflag.PullSysFlag; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++-import org.apache.rocketmq.common.BoundaryType; +++import org.apache.rocketmq.logging.org.slf4j.Logger; +++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.CommandCustomHeader; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.RPCHook; ++@@ -101,7 +104,10 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; ++ import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +++import org.apache.rocketmq.remoting.protocol.body.BatchAck; +++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; +++import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; ++ import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData; ++ import org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo; ++@@ -114,7 +120,6 @@ import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; ++ import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody; ++ import org.apache.rocketmq.remoting.protocol.body.GroupList; ++ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; ++-import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; ++ import org.apache.rocketmq.remoting.protocol.body.KVTable; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchResponseBody; ++@@ -196,6 +201,10 @@ import org.apache.rocketmq.remoting.protocol.header.UpdateGlobalWhiteAddrsConfig ++ import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.ViewBrokerStatsDataRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader; ++@@ -207,10 +216,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestH ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.PutKVConfigRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData; ++ import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; ++ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; ++@@ -221,8 +226,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden; ++ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; ++ import org.apache.rocketmq.remoting.rpchook.DynamicalExtFieldRPCHook; ++ import org.apache.rocketmq.remoting.rpchook.StreamTypeRPCHook; ++-import org.apache.rocketmq.logging.org.slf4j.Logger; ++-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ import static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS; ++ ++@@ -885,9 +888,77 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ final String addr, ++ final long timeOut, ++ final AckCallback ackCallback, ++- final AckMessageRequestHeader requestHeader // +++ final AckMessageRequestHeader requestHeader +++ ) throws RemotingException, MQBrokerException, InterruptedException { +++ ackMessageAsync(addr, timeOut, ackCallback, requestHeader, null); +++ } +++ +++ public void batchAckMessageAsync( +++ final String addr, +++ final long timeOut, +++ final AckCallback ackCallback, +++ final String topic, +++ final String consumerGroup, +++ final List extraInfoList +++ ) throws RemotingException, MQBrokerException, InterruptedException { +++ String brokerName = null; +++ Map batchAckMap = new HashMap<>(); +++ for (String extraInfo : extraInfoList) { +++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo); +++ if (brokerName == null) { +++ brokerName = ExtraInfoUtil.getBrokerName(extraInfoData); +++ } +++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + +++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" + +++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + +++ ExtraInfoUtil.getPopTime(extraInfoData); +++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { +++ BatchAck newBatchAck = new BatchAck(); +++ newBatchAck.setConsumerGroup(consumerGroup); +++ newBatchAck.setTopic(topic); +++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); +++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); +++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); +++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); +++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); +++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); +++ newBatchAck.setBitSet(new BitSet()); +++ return newBatchAck; +++ }); +++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); +++ } +++ +++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); +++ requestBody.setBrokerName(brokerName); +++ requestBody.setAcks(new ArrayList<>(batchAckMap.values())); +++ batchAckMessageAsync(addr, timeOut, ackCallback, requestBody); +++ } +++ +++ public void batchAckMessageAsync( +++ final String addr, +++ final long timeOut, +++ final AckCallback ackCallback, +++ final BatchAckMessageRequestBody requestBody ++ ) throws RemotingException, MQBrokerException, InterruptedException { ++- final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); +++ ackMessageAsync(addr, timeOut, ackCallback, null, requestBody); +++ } +++ +++ protected void ackMessageAsync( +++ final String addr, +++ final long timeOut, +++ final AckCallback ackCallback, +++ final AckMessageRequestHeader requestHeader, +++ final BatchAckMessageRequestBody requestBody +++ ) throws RemotingException, MQBrokerException, InterruptedException { +++ RemotingCommand request; +++ if (requestHeader != null) { +++ request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); +++ } else { +++ request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); +++ if (requestBody != null) { +++ request.setBody(requestBody.encode()); +++ } +++ } ++ this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) { ++ ++ @Override ++diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java ++index 496bd6da4..09c60c0b4 100644 ++--- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java +++++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java ++@@ -17,6 +17,7 @@ ++ ++ package org.apache.rocketmq.test.client.rmq; ++ +++import java.util.List; ++ import java.util.concurrent.CompletableFuture; ++ import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.consumer.AckCallback; ++@@ -140,6 +141,27 @@ public class RMQPopClient implements MQConsumer { ++ return future; ++ } ++ +++ public CompletableFuture batchAckMessageAsync(String brokerAddr, String topic, String consumerGroup, +++ List extraInfoList) { +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ this.mqClientAPI.batchAckMessageAsync(brokerAddr, DEFAULT_TIMEOUT, new AckCallback() { +++ @Override +++ public void onSuccess(AckResult ackResult) { +++ future.complete(ackResult); +++ } +++ +++ @Override +++ public void onException(Throwable e) { +++ future.completeExceptionally(e); +++ } +++ }, topic, consumerGroup, extraInfoList); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ ++ public CompletableFuture changeInvisibleTimeAsync(String brokerAddr, String brokerName, String topic, ++ String consumerGroup, String extraInfo, long invisibleTime) { ++ String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo); ++diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java ++index 952fbe3f5..2e29b95a5 100644 ++--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java +++++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java ++@@ -63,4 +63,10 @@ public class BasePopNormally extends BasePop { ++ brokerAddr, messageQueue, invisibleTime, maxNums, group, timeout, true, ++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); ++ } +++ +++ protected CompletableFuture popMessageAsync(long invisibleTime, int maxNums) { +++ return client.popMessageAsync( +++ brokerAddr, messageQueue, invisibleTime, maxNums, group, 3000, false, +++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); +++ } ++ } ++diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java ++new file mode 100644 ++index 000000000..ec9153ccc ++--- /dev/null +++++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java ++@@ -0,0 +1,159 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.test.client.consumer.pop; +++ +++import java.time.Duration; +++import java.util.ArrayList; +++import java.util.List; +++import java.util.concurrent.CompletableFuture; +++import java.util.concurrent.TimeUnit; +++import java.util.concurrent.atomic.AtomicInteger; +++import java.util.function.Supplier; +++import org.apache.rocketmq.client.consumer.AckResult; +++import org.apache.rocketmq.client.consumer.AckStatus; +++import org.apache.rocketmq.client.consumer.PopResult; +++import org.apache.rocketmq.client.consumer.PopStatus; +++import org.apache.rocketmq.common.attribute.CQType; +++import org.apache.rocketmq.common.attribute.TopicMessageType; +++import org.apache.rocketmq.common.constant.ConsumeInitMode; +++import org.apache.rocketmq.common.filter.ExpressionType; +++import org.apache.rocketmq.common.message.MessageConst; +++import org.apache.rocketmq.common.message.MessageExt; +++import org.apache.rocketmq.common.message.MessageQueue; +++import org.apache.rocketmq.test.base.IntegrationTestBase; +++import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; +++import org.apache.rocketmq.test.client.rmq.RMQPopClient; +++import org.apache.rocketmq.test.util.MQRandomUtils; +++import org.junit.After; +++import org.junit.Before; +++import org.junit.Test; +++ +++import static org.awaitility.Awaitility.await; +++import static org.junit.Assert.assertEquals; +++ +++public class BatchAckIT extends BasePop { +++ +++ protected String topic; +++ protected String group; +++ protected RMQNormalProducer producer = null; +++ protected RMQPopClient client = null; +++ protected String brokerAddr; +++ protected MessageQueue messageQueue; +++ +++ @Before +++ public void setUp() { +++ brokerAddr = brokerController1.getBrokerAddr(); +++ topic = MQRandomUtils.getRandomTopic(); +++ group = initConsumerGroup(); +++ IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL); +++ producer = getProducer(NAMESRV_ADDR, topic); +++ client = getRMQPopClient(); +++ messageQueue = new MessageQueue(topic, BROKER1_NAME, -1); +++ } +++ +++ @After +++ public void tearDown() { +++ shutdown(); +++ } +++ +++ @Test +++ public void testBatchAckNormallyWithPopBuffer() throws Throwable { +++ brokerController1.getBrokerConfig().setEnablePopBufferMerge(true); +++ brokerController2.getBrokerConfig().setEnablePopBufferMerge(true); +++ +++ testBatchAck(() -> { +++ try { +++ return popMessageAsync().get(); +++ } catch (Exception e) { +++ throw new RuntimeException(e); +++ } +++ }); +++ } +++ +++ @Test +++ public void testBatchAckNormallyWithOutPopBuffer() throws Throwable { +++ brokerController1.getBrokerConfig().setEnablePopBufferMerge(false); +++ brokerController2.getBrokerConfig().setEnablePopBufferMerge(false); +++ +++ testBatchAck(() -> { +++ try { +++ return popMessageAsync().get(); +++ } catch (Exception e) { +++ throw new RuntimeException(e); +++ } +++ }); +++ } +++ +++ @Test +++ public void testBatchAckOrderly() throws Throwable { +++ testBatchAck(() -> { +++ try { +++ return popMessageOrderlyAsync().get(); +++ } catch (Exception e) { +++ throw new RuntimeException(e); +++ } +++ }); +++ } +++ +++ public void testBatchAck(Supplier popResultSupplier) throws Throwable { +++ // Send 10 messages but do not ack, let them enter the retry topic +++ producer.send(10); +++ AtomicInteger firstMsgRcvNum = new AtomicInteger(); +++ await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> { +++ PopResult popResult = popResultSupplier.get(); +++ if (popResult.getPopStatus().equals(PopStatus.FOUND)) { +++ firstMsgRcvNum.addAndGet(popResult.getMsgFoundList().size()); +++ } +++ assertEquals(10, firstMsgRcvNum.get()); +++ }); +++ // sleep 6s, expect messages to enter the retry topic +++ TimeUnit.SECONDS.sleep(6); +++ +++ producer.send(20); +++ List extraInfoList = new ArrayList<>(); +++ await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> { +++ PopResult popResult = popResultSupplier.get(); +++ if (popResult.getPopStatus().equals(PopStatus.FOUND)) { +++ for (MessageExt messageExt : popResult.getMsgFoundList()) { +++ extraInfoList.add(messageExt.getProperty(MessageConst.PROPERTY_POP_CK)); +++ } +++ } +++ assertEquals(30, extraInfoList.size()); +++ }); +++ +++ AckResult ackResult = client.batchAckMessageAsync(brokerAddr, topic, group, extraInfoList).get(); +++ assertEquals(AckStatus.OK, ackResult.getStatus()); +++ +++ // sleep 6s, expected that messages that have been acked will not be re-consumed +++ TimeUnit.SECONDS.sleep(6); +++ PopResult popResult = popResultSupplier.get(); +++ assertEquals(PopStatus.POLLING_NOT_FOUND, popResult.getPopStatus()); +++ } +++ +++ private CompletableFuture popMessageAsync() { +++ return client.popMessageAsync( +++ brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false, +++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); +++ } +++ +++ private CompletableFuture popMessageOrderlyAsync() { +++ return client.popMessageAsync( +++ brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false, +++ ConsumeInitMode.MIN, true, ExpressionType.TAG, "*", null); +++ } +++} ++-- ++2.32.0.windows.2 ++ ++ ++From cc16a1b51216e1e80c22011b8b01e060bb4af8b3 Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Tue, 22 Aug 2023 10:42:25 +0800 ++Subject: [PATCH 7/7] Set table reference the same object for ++ setSubscriptionGroupTable method (#7204) ++ ++--- ++ .../broker/subscription/SubscriptionGroupManager.java | 5 +---- ++ 1 file changed, 1 insertion(+), 4 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java ++index 74e39c0fe..e63b93058 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java ++@@ -341,10 +341,7 @@ public class SubscriptionGroupManager extends ConfigManager { ++ ++ ++ public void setSubscriptionGroupTable(ConcurrentMap subscriptionGroupTable) { ++- this.subscriptionGroupTable.clear(); ++- for (String key : subscriptionGroupTable.keySet()) { ++- putSubscriptionGroupConfig(subscriptionGroupTable.get(key)); ++- } +++ this.subscriptionGroupTable = subscriptionGroupTable; ++ } ++ ++ public boolean containsSubscriptionGroup(String group) { ++-- ++2.32.0.windows.2 ++ +diff --git a/patch011-backport-optimize-opentelemetry-metric-config.patch b/patch011-backport-optimize-opentelemetry-metric-config.patch +new file mode 100644 +index 000000000..b59d7e2c7 +--- /dev/null ++++ b/patch011-backport-optimize-opentelemetry-metric-config.patch +@@ -0,0 +1,2081 @@ ++From 744167bd01fab6821b4d5ae1794dc845153d5156 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Wed, 23 Aug 2023 08:32:17 +0800 ++Subject: [PATCH 1/7] [ISSUE #7142] Add command `RocksDBConfigToJson` to ++ inspect rocksdb content (#7180) ++ ++* feat: add command `RocksDBConfigToJson` to inspect rocksdb content ++ ++Signed-off-by: Ziy1-Tan ++ ++* refactor: fix style ++ ++--------- ++ ++Signed-off-by: Ziy1-Tan ++Co-authored-by: Ziy1-Tan ++--- ++ .../tools/command/MQAdminStartup.java | 2 + ++ .../metadata/RocksDBConfigToJsonCommand.java | 118 ++++++++++++++++++ ++ .../metadata/KvConfigToJsonCommandTest.java | 65 ++++++++++ ++ 3 files changed, 185 insertions(+) ++ create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++ create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++index 890125ca0..324aa1856 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++@@ -80,6 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; ++ import org.apache.rocketmq.tools.command.message.SendMessageCommand; +++import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; ++ import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; ++ import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; ++ import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; ++@@ -211,6 +212,7 @@ public class MQAdminStartup { ++ ++ initCommand(new ClusterListSubCommand()); ++ initCommand(new TopicListSubCommand()); +++ initCommand(new RocksDBConfigToJsonCommand()); ++ ++ initCommand(new UpdateKvConfigCommand()); ++ initCommand(new DeleteKvConfigCommand()); ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++new file mode 100644 ++index 000000000..3053f4684 ++--- /dev/null +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++@@ -0,0 +1,118 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.metadata; +++ +++import com.alibaba.fastjson.JSONObject; +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.Option; +++import org.apache.commons.cli.Options; +++import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.common.config.RocksDBConfigManager; +++import org.apache.rocketmq.common.utils.DataConverter; +++import org.apache.rocketmq.remoting.RPCHook; +++import org.apache.rocketmq.tools.command.SubCommand; +++import org.apache.rocketmq.tools.command.SubCommandException; +++ +++import java.io.File; +++import java.util.HashMap; +++import java.util.Map; +++ +++public class RocksDBConfigToJsonCommand implements SubCommand { +++ private static final String TOPICS_JSON_CONFIG = "topics"; +++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; +++ +++ @Override +++ public String commandName() { +++ return "rocksDBConfigToJson"; +++ } +++ +++ @Override +++ public String commandDesc() { +++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; +++ } +++ +++ @Override +++ public Options buildCommandlineOptions(Options options) { +++ Option pathOption = new Option("p", "path", true, +++ "Absolute path to the metadata directory"); +++ pathOption.setRequired(true); +++ options.addOption(pathOption); +++ +++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + +++ "topics/subscriptionGroups"); +++ configTypeOption.setRequired(true); +++ options.addOption(configTypeOption); +++ +++ return options; +++ } +++ +++ @Override +++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { +++ String path = commandLine.getOptionValue("path").trim(); +++ if (StringUtils.isEmpty(path) || !new File(path).exists()) { +++ System.out.print("Rocksdb path is invalid.\n"); +++ return; +++ } +++ +++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); +++ +++ final long memTableFlushInterval = 60 * 60 * 1000L; +++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); +++ try { +++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { +++ // for topics.json +++ final Map topicsJsonConfig = new HashMap<>(); +++ final Map topicConfigTable = new HashMap<>(); +++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { +++ final String topic = new String(key, DataConverter.charset); +++ final String topicConfig = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig); +++ topicConfigTable.put(topic, jsonObject); +++ }); +++ +++ if (isLoad) { +++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); +++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); +++ System.out.print(topicsJsonStr + "\n"); +++ return; +++ } +++ } +++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { +++ // for subscriptionGroup.json +++ final Map subscriptionGroupJsonConfig = new HashMap<>(); +++ final Map subscriptionGroupTable = new HashMap<>(); +++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { +++ final String subscriptionGroup = new String(key, DataConverter.charset); +++ final String subscriptionGroupConfig = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); +++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); +++ }); +++ +++ if (isLoad) { +++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", +++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); +++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); +++ System.out.print(subscriptionGroupJsonStr + "\n"); +++ return; +++ } +++ } +++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); +++ } finally { +++ kvConfigManager.stop(); +++ } +++ } +++} ++diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++new file mode 100644 ++index 000000000..b2f66c7b0 ++--- /dev/null +++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++@@ -0,0 +1,65 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.metadata; +++ +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.DefaultParser; +++import org.apache.commons.cli.Options; +++import org.apache.rocketmq.srvutil.ServerUtil; +++import org.apache.rocketmq.tools.command.SubCommandException; +++import org.junit.Test; +++ +++import java.io.File; +++ +++import static org.assertj.core.api.Assertions.assertThat; +++ +++public class KvConfigToJsonCommandTest { +++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; +++ +++ @Test +++ public void testExecute() throws SubCommandException { +++ { +++ String[] cases = new String[]{"topics", "subscriptionGroups"}; +++ for (String c : cases) { +++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ Options options = ServerUtil.buildCommandlineOptions(new Options()); +++ String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c}; +++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, +++ cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.execute(commandLine, options, null); +++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c); +++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c); +++ } +++ } +++ // invalid cases +++ { +++ String[][] cases = new String[][]{ +++ {"-p " + BASE_PATH + "tmpPath", "-t topics"}, +++ {"-p ", "-t topics"}, +++ {"-p " + BASE_PATH + "topics", "-t invalid_type"} +++ }; +++ +++ for (String[] c : cases) { +++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ Options options = ServerUtil.buildCommandlineOptions(new Options()); +++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, +++ cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.execute(commandLine, options, null); +++ } +++ } +++ } +++} ++-- ++2.32.0.windows.2 ++ ++ ++From bdede35db365a49b211cdc249c68b0f60a3df46d Mon Sep 17 00:00:00 2001 ++From: mxsm ++Date: Wed, 23 Aug 2023 08:34:56 +0800 ++Subject: [PATCH 2/7] [ISSUE #7124] Fix the typos in the code comments (#7125) ++ ++--- ++ .../apache/rocketmq/broker/processor/ReplyMessageProcessor.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++index b2db356c8..d3bb048f7 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++@@ -234,7 +234,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor { ++ } else { ++ response.setCode(ResponseCode.SUCCESS); ++ response.setRemark(null); ++- //set to zore to avoid client decoding exception +++ //set to zero to avoid client decoding exception ++ responseHeader.setMsgId("0"); ++ responseHeader.setQueueId(queueIdInt); ++ responseHeader.setQueueOffset(0L); ++-- ++2.32.0.windows.2 ++ ++ ++From 9bb73b9a38548b99ac5126c40380c3c2e7fc586e Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 23 Aug 2023 09:46:27 +0800 ++Subject: [PATCH 3/7] [#ISSUE 7222] Bug fix and refactoring of the Indexfile in ++ tiered storage (#7224) ++ ++--- ++ .../tieredstore/file/TieredIndexFile.java | 38 +++++++-- ++ .../tieredstore/file/TieredIndexFileTest.java | 84 +++++-------------- ++ 2 files changed, 52 insertions(+), 70 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++index 50beb01ae..eda5e0106 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ +++import com.google.common.annotations.VisibleForTesting; ++ import java.io.File; ++ import java.io.IOException; ++ import java.nio.ByteBuffer; ++@@ -99,7 +100,7 @@ public class TieredIndexFile { ++ this::doScheduleTask, 10, 10, TimeUnit.SECONDS); ++ } ++ ++- private void doScheduleTask() { +++ protected void doScheduleTask() { ++ try { ++ curFileLock.lock(); ++ try { ++@@ -145,6 +146,11 @@ public class TieredIndexFile { ++ } ++ } ++ +++ @VisibleForTesting +++ public MappedFile getPreMappedFile() { +++ return preMappedFile; +++ } +++ ++ private void initFile() throws IOException { ++ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); ++ initIndexFileHeader(curMappedFile); ++@@ -156,19 +162,26 @@ public class TieredIndexFile { ++ ++ if (isFileSealed(curMappedFile)) { ++ if (preFileExists) { ++- preFile.delete(); +++ if (preFile.delete()) { +++ logger.info("Pre IndexFile deleted success", preFilepath); +++ } else { +++ logger.error("Pre IndexFile deleted failed", preFilepath); +++ } ++ } ++ boolean rename = curMappedFile.renameTo(preFilepath); ++ if (rename) { ++ preMappedFile = curMappedFile; ++ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); +++ initIndexFileHeader(curMappedFile); ++ preFileExists = true; ++ } ++ } +++ ++ if (preFileExists) { ++ synchronized (TieredIndexFile.class) { ++ if (inflightCompactFuture.isDone()) { ++- inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(new CompactTask(storeConfig, preMappedFile, flatFile), null); +++ inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit( +++ new CompactTask(storeConfig, preMappedFile, flatFile), null); ++ } ++ } ++ } ++@@ -261,7 +274,8 @@ public class TieredIndexFile { ++ } ++ } ++ ++- public CompletableFuture>> queryAsync(String topic, String key, long beginTime, long endTime) { +++ public CompletableFuture>> queryAsync(String topic, String key, long beginTime, +++ long endTime) { ++ int hashCode = indexKeyHashMethod(buildKey(topic, key)); ++ int slotPosition = hashCode % maxHashSlotNum; ++ List fileSegmentList = flatFile.getFileListByTime(beginTime, endTime); ++@@ -355,7 +369,7 @@ public class TieredIndexFile { ++ private final int fileMaxSize; ++ private MappedFile originFile; ++ private TieredFlatFile fileQueue; ++- private final MappedFile compactFile; +++ private MappedFile compactFile; ++ ++ public CompactTask(TieredMessageStoreConfig storeConfig, MappedFile originFile, ++ TieredFlatFile fileQueue) throws IOException { ++@@ -381,6 +395,17 @@ public class TieredIndexFile { ++ } catch (Throwable throwable) { ++ logger.error("TieredIndexFile#compactTask: compact index file failed:", throwable); ++ } +++ +++ try { +++ if (originFile != null) { +++ originFile.destroy(-1); +++ } +++ if (compactFile != null) { +++ compactFile.destroy(-1); +++ } +++ } catch (Throwable throwable) { +++ logger.error("TieredIndexFile#compactTask: destroy index file failed:", throwable); +++ } ++ } ++ ++ public void compact() { ++@@ -396,6 +421,8 @@ public class TieredIndexFile { ++ fileQueue.commit(true); ++ compactFile.destroy(-1); ++ originFile.destroy(-1); +++ compactFile = null; +++ originFile = null; ++ } ++ ++ private void buildCompactFile() { ++@@ -414,6 +441,7 @@ public class TieredIndexFile { ++ if (slotValue != -1) { ++ int indexTotalSize = 0; ++ int indexPosition = slotValue; +++ ++ while (indexPosition >= 0 && indexPosition < maxIndexNum) { ++ int indexOffset = INDEX_FILE_HEADER_SIZE + maxHashSlotNum * INDEX_FILE_HASH_SLOT_SIZE ++ + indexPosition * INDEX_FILE_HASH_ORIGIN_INDEX_SIZE; ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++index 7ef49578d..262d6645b 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++@@ -19,9 +19,8 @@ package org.apache.rocketmq.tieredstore.file; ++ import com.sun.jna.Platform; ++ import java.io.IOException; ++ import java.nio.ByteBuffer; +++import java.time.Duration; ++ import java.util.List; ++-import java.util.concurrent.TimeUnit; ++-import org.apache.commons.lang3.SystemUtils; ++ import org.apache.commons.lang3.tuple.Pair; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; ++@@ -31,9 +30,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ import org.awaitility.Awaitility; ++ import org.junit.After; ++ import org.junit.Assert; ++-import org.junit.Assume; ++ import org.junit.Before; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++ public class TieredIndexFileTest { ++@@ -45,11 +42,12 @@ public class TieredIndexFileTest { ++ @Before ++ public void setUp() { ++ storeConfig = new TieredMessageStoreConfig(); +++ storeConfig.setBrokerName("IndexFileBroker"); ++ storeConfig.setStorePathRootDir(storePath); ++- storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment"); ++- storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2); ++- storeConfig.setTieredStoreIndexFileMaxIndexNum(3); ++- mq = new MessageQueue("TieredIndexFileTest", storeConfig.getBrokerName(), 1); +++ storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment"); +++ storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5); +++ storeConfig.setTieredStoreIndexFileMaxIndexNum(20); +++ mq = new MessageQueue("IndexFileTest", storeConfig.getBrokerName(), 1); ++ TieredStoreUtil.getMetadataStore(storeConfig); ++ TieredStoreExecutor.init(); ++ } ++@@ -61,77 +59,33 @@ public class TieredIndexFileTest { ++ TieredStoreExecutor.shutdown(); ++ } ++ ++- @Ignore ++ @Test ++ public void testAppendAndQuery() throws IOException, ClassNotFoundException, NoSuchMethodException { ++ if (Platform.isWindows()) { ++ return; ++ } ++ ++- // skip this test on windows ++- Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); ++- ++ TieredFileAllocator fileQueueFactory = new TieredFileAllocator(storeConfig); ++ TieredIndexFile indexFile = new TieredIndexFile(fileQueueFactory, storePath); +++ ++ indexFile.append(mq, 0, "key3", 3, 300, 1000); ++ indexFile.append(mq, 0, "key2", 2, 200, 1100); ++ indexFile.append(mq, 0, "key1", 1, 100, 1200); ++ ++- Awaitility.waitAtMost(5, TimeUnit.SECONDS) ++- .until(() -> { ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++- if (indexList.size() != 1) { ++- return false; ++- } ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 2, indexBuffer.remaining()); ++- ++- Assert.assertEquals(1, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++- ++- Assert.assertEquals(3, indexBuffer.getLong(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4)); ++- Assert.assertEquals(300, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8 + 4)); ++- return true; ++- }); ++- ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- ++- Awaitility.waitAtMost(5, TimeUnit.SECONDS) ++- .until(() -> { ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1300, 1300).join(); ++- if (indexList.size() != 1) { ++- return false; ++- } ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); ++- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++- return true; ++- }); ++- ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1300, 1300).join(); +++ // do not do schedule task here +++ TieredStoreExecutor.shutdown(); +++ List> indexList = +++ indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++ Assert.assertEquals(0, indexList.size()); ++ ++- indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1200, 1300).join(); ++- Assert.assertEquals(2, indexList.size()); ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); ++- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +++ // do compaction once +++ TieredStoreExecutor.init(); +++ storeConfig.setTieredStoreIndexFileRollingIdleInterval(0); +++ indexFile.doScheduleTask(); +++ Awaitility.await().atMost(Duration.ofSeconds(10)) +++ .until(() -> !indexFile.getPreMappedFile().getFile().exists()); ++ ++- indexBuffer = indexList.get(1).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE, indexBuffer.remaining()); ++- Assert.assertEquals(2, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); +++ Assert.assertEquals(1, indexList.size()); ++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 69c26d3d29cde7b4484ecd112ab9224f9f42bf45 Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Wed, 23 Aug 2023 10:27:52 +0800 ++Subject: [PATCH 4/7] [ISSUE #7228] Converge the use of some important ++ variables for some class ++ ++--- ++ .../apache/rocketmq/store/ConsumeQueue.java | 16 ++++++------ ++ .../rocketmq/store/MappedFileQueue.java | 26 +++++++++++-------- ++ .../store/MultiPathMappedFileQueue.java | 4 +-- ++ 3 files changed, 24 insertions(+), 22 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++index a0b886eb0..56bee2af3 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++@@ -145,7 +145,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ if (offset >= 0 && size > 0) { ++ mappedFileOffset = i + CQ_STORE_UNIT_SIZE; ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++ } ++@@ -409,7 +409,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ int logicFileSize = this.mappedFileSize; ++ ++- this.maxPhysicOffset = phyOffset; +++ this.setMaxPhysicOffset(phyOffset); ++ long maxExtAddr = 1; ++ boolean shouldDeleteFile = false; ++ while (true) { ++@@ -435,7 +435,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ mappedFile.setWrotePosition(pos); ++ mappedFile.setCommittedPosition(pos); ++ mappedFile.setFlushedPosition(pos); ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ // This maybe not take effect, when not every consume queue has extend file. ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++@@ -453,7 +453,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ mappedFile.setWrotePosition(pos); ++ mappedFile.setCommittedPosition(pos); ++ mappedFile.setFlushedPosition(pos); ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++ } ++@@ -881,8 +881,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, ++ final long cqOffset) { ++ ++- if (offset + size <= this.maxPhysicOffset) { ++- log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset); +++ if (offset + size <= this.getMaxPhysicOffset()) { +++ log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", this.getMaxPhysicOffset(), offset); ++ return true; ++ } ++ ++@@ -926,7 +926,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ); ++ } ++ } ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ return mappedFile.appendMessage(this.byteBufferIndex.array()); ++ } ++ return false; ++@@ -1130,7 +1130,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ @Override ++ public void destroy() { ++- this.maxPhysicOffset = -1; +++ this.setMaxPhysicOffset(-1); ++ this.minLogicOffset = 0; ++ this.mappedFileQueue.destroy(); ++ if (isExtReadEnable()) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++index 0bc70642f..32b90d14f 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++@@ -285,7 +285,7 @@ public class MappedFileQueue implements Swappable { ++ if (this.mappedFiles.isEmpty()) ++ return 0; ++ ++- long committed = this.flushedWhere; +++ long committed = this.getFlushedWhere(); ++ if (committed != 0) { ++ MappedFile mappedFile = this.getLastMappedFile(0, false); ++ if (mappedFile != null) { ++@@ -442,11 +442,11 @@ public class MappedFileQueue implements Swappable { ++ } ++ ++ public long remainHowManyDataToCommit() { ++- return getMaxWrotePosition() - committedWhere; +++ return getMaxWrotePosition() - getCommittedWhere(); ++ } ++ ++ public long remainHowManyDataToFlush() { ++- return getMaxOffset() - flushedWhere; +++ return getMaxOffset() - this.getFlushedWhere(); ++ } ++ ++ public void deleteLastMappedFile() { ++@@ -616,15 +616,15 @@ public class MappedFileQueue implements Swappable { ++ ++ public boolean flush(final int flushLeastPages) { ++ boolean result = true; ++- MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0); +++ MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0); ++ if (mappedFile != null) { ++ long tmpTimeStamp = mappedFile.getStoreTimestamp(); ++ int offset = mappedFile.flush(flushLeastPages); ++ long where = mappedFile.getFileFromOffset() + offset; ++- result = where == this.flushedWhere; ++- this.flushedWhere = where; +++ result = where == this.getFlushedWhere(); +++ this.setFlushedWhere(where); ++ if (0 == flushLeastPages) { ++- this.storeTimestamp = tmpTimeStamp; +++ this.setStoreTimestamp(tmpTimeStamp); ++ } ++ } ++ ++@@ -633,12 +633,12 @@ public class MappedFileQueue implements Swappable { ++ ++ public synchronized boolean commit(final int commitLeastPages) { ++ boolean result = true; ++- MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0); +++ MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0); ++ if (mappedFile != null) { ++ int offset = mappedFile.commit(commitLeastPages); ++ long where = mappedFile.getFileFromOffset() + offset; ++- result = where == this.committedWhere; ++- this.committedWhere = where; +++ result = where == this.getCommittedWhere(); +++ this.setCommittedWhere(where); ++ } ++ ++ return result; ++@@ -763,7 +763,7 @@ public class MappedFileQueue implements Swappable { ++ mf.destroy(1000 * 3); ++ } ++ this.mappedFiles.clear(); ++- this.flushedWhere = 0; +++ this.setFlushedWhere(0); ++ ++ // delete parent directory ++ File file = new File(storePath); ++@@ -848,6 +848,10 @@ public class MappedFileQueue implements Swappable { ++ return storeTimestamp; ++ } ++ +++ public void setStoreTimestamp(long storeTimestamp) { +++ this.storeTimestamp = storeTimestamp; +++ } +++ ++ public List getMappedFiles() { ++ return mappedFiles; ++ } ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++index 8f5af9438..8ff050dfe 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.store; ++ ++- ++ import java.util.Arrays; ++ import java.util.HashSet; ++ import java.util.Set; ++@@ -113,8 +112,7 @@ public class MultiPathMappedFileQueue extends MappedFileQueue { ++ mf.destroy(1000 * 3); ++ } ++ this.mappedFiles.clear(); ++- this.flushedWhere = 0; ++- +++ this.setFlushedWhere(0); ++ ++ Set storePathSet = getPaths(); ++ storePathSet.addAll(getReadonlyPaths()); ++-- ++2.32.0.windows.2 ++ ++ ++From 3884f595949462044c5cb3c236199bc1d7ad2341 Mon Sep 17 00:00:00 2001 ++From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= ++ ++Date: Wed, 23 Aug 2023 11:10:30 +0800 ++Subject: [PATCH 5/7] [ISSUE #7149] When creating and updating Topic, there ++ will be problems with permission settings (#7151) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings ++ ++* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings ++ ++* [issue#7249] ++ ++--------- ++ ++Co-authored-by: 十真 ++--- ++ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 3 ++- ++ 1 file changed, 2 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 13f9d002b..e8f943702 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1733,7 +1733,8 @@ public class BrokerController { ++ new TopicConfig(topicConfig.getTopicName(), ++ topicConfig.getReadQueueNums(), ++ topicConfig.getWriteQueueNums(), ++- this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); +++ topicConfig.getPerm() +++ & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); ++ } else { ++ registerTopicConfig = new TopicConfig(topicConfig); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 017ad110475e8024585327b44f47e5e97aabc63b Mon Sep 17 00:00:00 2001 ++From: echooymxq ++Date: Wed, 23 Aug 2023 11:11:42 +0800 ++Subject: [PATCH 6/7] [ISSUE #7219] Fix Concurrent modify syncStateSet and Mark ++ synchronizing frequently when shrink. (#7220) ++ ++--- ++ .../broker/controller/ReplicasManager.java | 29 ++++++++++--------- ++ .../ha/autoswitch/AutoSwitchHAService.java | 21 ++++++++------ ++ 2 files changed, 28 insertions(+), 22 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++index abae7cdb0..37c82e434 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++@@ -542,7 +542,7 @@ public class ReplicasManager { ++ this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId()); ++ this.tempBrokerMetadata.clear(); ++ this.brokerControllerId = this.brokerMetadata.getBrokerId(); ++- this.haService.setBrokerControllerId(this.brokerControllerId); +++ this.haService.setLocalBrokerId(this.brokerControllerId); ++ return true; ++ } catch (Exception e) { ++ LOGGER.error("fail to create metadata file", e); ++@@ -594,7 +594,7 @@ public class ReplicasManager { ++ if (this.brokerMetadata.isLoaded()) { ++ this.registerState = RegisterState.CREATE_METADATA_FILE_DONE; ++ this.brokerControllerId = brokerMetadata.getBrokerId(); ++- this.haService.setBrokerControllerId(this.brokerControllerId); +++ this.haService.setLocalBrokerId(this.brokerControllerId); ++ return; ++ } ++ // 2. check if temp metadata exist ++@@ -735,23 +735,26 @@ public class ReplicasManager { ++ if (this.checkSyncStateSetTaskFuture != null) { ++ this.checkSyncStateSetTaskFuture.cancel(false); ++ } ++- this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(() -> { ++- checkSyncStateSetAndDoReport(); ++- }, 3 * 1000, this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); +++ this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000, +++ this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); ++ } ++ ++ private void checkSyncStateSetAndDoReport() { ++- final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); ++- newSyncStateSet.add(this.brokerControllerId); ++- synchronized (this) { ++- if (this.syncStateSet != null) { ++- // Check if syncStateSet changed ++- if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { ++- return; +++ try { +++ final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); +++ newSyncStateSet.add(this.brokerControllerId); +++ synchronized (this) { +++ if (this.syncStateSet != null) { +++ // Check if syncStateSet changed +++ if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { +++ return; +++ } ++ } ++ } +++ doReportSyncStateSetChanged(newSyncStateSet); +++ } catch (Exception e) { +++ LOGGER.error("Check syncStateSet error", e); ++ } ++- doReportSyncStateSetChanged(newSyncStateSet); ++ } ++ ++ private void doReportSyncStateSetChanged(Set newSyncStateSet) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++index 6dc734e0c..d5393fdca 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++@@ -41,6 +41,7 @@ import java.nio.channels.SocketChannel; ++ import java.util.ArrayList; ++ import java.util.HashSet; ++ import java.util.List; +++import java.util.Iterator; ++ import java.util.Map; ++ import java.util.Objects; ++ import java.util.Set; ++@@ -73,7 +74,7 @@ public class AutoSwitchHAService extends DefaultHAService { ++ private EpochFileCache epochCache; ++ private AutoSwitchHAClient haClient; ++ ++- private Long brokerControllerId = null; +++ private Long localBrokerId = null; ++ ++ public AutoSwitchHAService() { ++ } ++@@ -287,9 +288,11 @@ public class AutoSwitchHAService extends DefaultHAService { ++ ++ // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable, ++ // it means that the broker has not connected. ++- for (Long slaveBrokerId : newSyncStateSet) { ++- if (!this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { ++- newSyncStateSet.remove(slaveBrokerId); +++ Iterator iterator = newSyncStateSet.iterator(); +++ while (iterator.hasNext()) { +++ Long slaveBrokerId = iterator.next(); +++ if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { +++ iterator.remove(); ++ isSyncStateSetChanged = true; ++ } ++ } ++@@ -419,7 +422,7 @@ public class AutoSwitchHAService extends DefaultHAService { ++ // To avoid the syncStateSet is not consistent with connectionList. ++ // Fix issue: https://github.com/apache/rocketmq/issues/6662 ++ for (Long syncId : currentSyncStateSet) { ++- if (!idList.contains(syncId) && this.brokerControllerId != null && !Objects.equals(syncId, this.brokerControllerId)) { +++ if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) { ++ LOGGER.warn("Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.", syncId); ++ // Without check and re-compute, return the confirmOffset's value directly. ++ return this.defaultMessageStore.getConfirmOffsetDirectly(); ++@@ -545,12 +548,12 @@ public class AutoSwitchHAService extends DefaultHAService { ++ return this.epochCache.getAllEntries(); ++ } ++ ++- public Long getBrokerControllerId() { ++- return brokerControllerId; +++ public Long getLocalBrokerId() { +++ return localBrokerId; ++ } ++ ++- public void setBrokerControllerId(Long brokerControllerId) { ++- this.brokerControllerId = brokerControllerId; +++ public void setLocalBrokerId(Long localBrokerId) { +++ this.localBrokerId = localBrokerId; ++ } ++ ++ class AutoSwitchAcceptSocketService extends AcceptSocketService { ++-- ++2.32.0.windows.2 ++ ++ ++From 77e8e54b37c3fc3ea0beffc1ace6f5bf20af10d9 Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Wed, 23 Aug 2023 15:56:39 +0800 ++Subject: [PATCH 7/7] [ISSUE #7223] Support batch ack for grpc client in proxy ++ (#7225) ++ ++--- ++ .../client/impl/mqclient/MQClientAPIExt.java | 26 +++ ++ .../rocketmq/proxy/config/ProxyConfig.java | 10 + ++ .../grpc/v2/consumer/AckMessageActivity.java | 136 ++++++++--- ++ .../proxy/processor/AbstractProcessor.java | 4 +- ++ .../proxy/processor/BatchAckResult.java | 53 +++++ ++ .../proxy/processor/ConsumerProcessor.java | 64 +++++ ++ .../processor/DefaultMessagingProcessor.java | 7 + ++ .../proxy/processor/MessagingProcessor.java | 18 ++ ++ .../message/ClusterMessageService.java | 16 +- ++ .../service/message/LocalMessageService.java | 58 +++++ ++ .../proxy/service/message/MessageService.java | 8 + ++ .../service/message/ReceiptHandleMessage.java | 39 ++++ ++ .../v2/consumer/AckMessageActivityTest.java | 221 +++++++++++++++--- ++ .../proxy/processor/BaseProcessorTest.java | 18 +- ++ .../processor/ConsumerProcessorTest.java | 115 +++++++++ ++ .../service/mqclient/MQClientAPIExtTest.java | 12 + ++ 16 files changed, 728 insertions(+), 77 deletions(-) ++ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++index fb8f8d11f..d7c8ef8d9 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++@@ -306,6 +306,32 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ return future; ++ } ++ +++ public CompletableFuture batchAckMessageAsync( +++ String brokerAddr, +++ String topic, +++ String consumerGroup, +++ List extraInfoList, +++ long timeoutMillis +++ ) { +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() { +++ @Override +++ public void onSuccess(AckResult ackResult) { +++ future.complete(ackResult); +++ } +++ +++ @Override +++ public void onException(Throwable t) { +++ future.completeExceptionally(t); +++ } +++ }, topic, consumerGroup, extraInfoList); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ ++ public CompletableFuture changeInvisibleTimeAsync( ++ String brokerAddr, ++ String brokerName, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 39caaa0d9..76a243919 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -250,6 +250,8 @@ public class ProxyConfig implements ConfigFile { ++ private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000; ++ private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000; ++ +++ private boolean enableBatchAck = false; +++ ++ @Override ++ public void initData() { ++ parseDelayLevel(); ++@@ -1379,4 +1381,12 @@ public class ProxyConfig implements ConfigFile { ++ public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) { ++ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; ++ } +++ +++ public boolean isEnableBatchAck() { +++ return enableBatchAck; +++ } +++ +++ public void setEnableBatchAck(boolean enableBatchAck) { +++ this.enableBatchAck = enableBatchAck; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++index 9a3a77201..97c716c8f 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++@@ -31,12 +31,15 @@ import org.apache.rocketmq.client.consumer.AckStatus; ++ import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; +++import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; ++ import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; ++ import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +++import org.apache.rocketmq.proxy.processor.BatchAckResult; ++ import org.apache.rocketmq.proxy.processor.MessagingProcessor; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ ++ public class AckMessageActivity extends AbstractMessingActivity { ++ ++@@ -50,60 +53,98 @@ public class AckMessageActivity extends AbstractMessingActivity { ++ ++ try { ++ validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); ++- ++- CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; ++- for (int i = 0; i < request.getEntriesCount(); i++) { ++- futures[i] = processAckMessage(ctx, request, request.getEntries(i)); +++ String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); +++ String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()); +++ if (ConfigurationManager.getProxyConfig().isEnableBatchAck()) { +++ future = ackMessageInBatch(ctx, group, topic, request); +++ } else { +++ future = ackMessageOneByOne(ctx, group, topic, request); ++ } ++- CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { ++- if (throwable != null) { ++- future.completeExceptionally(throwable); ++- return; ++- } +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ +++ protected CompletableFuture ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) { +++ List handleMessageList = new ArrayList<>(request.getEntriesCount()); ++ +++ for (AckMessageEntry ackMessageEntry : request.getEntriesList()) { +++ String handleString = getHandleString(ctx, group, request, ackMessageEntry); +++ handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId())); +++ } +++ return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic) +++ .thenApply(batchAckResultList -> { +++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder(); ++ Set responseCodes = new HashSet<>(); ++- List entryList = new ArrayList<>(); ++- for (CompletableFuture entryFuture : futures) { ++- AckMessageResultEntry entryResult = entryFuture.join(); ++- responseCodes.add(entryResult.getStatus().getCode()); ++- entryList.add(entryResult); ++- } ++- AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() ++- .addAllEntries(entryList); ++- if (responseCodes.size() > 1) { ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); ++- } else if (responseCodes.size() == 1) { ++- Code code = responseCodes.stream().findAny().get(); ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); ++- } else { ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult); +++ responseBuilder.addEntries(entry); +++ responseCodes.add(entry.getStatus().getCode()); ++ } ++- future.complete(responseBuilder.build()); +++ setAckResponseStatus(responseBuilder, responseCodes); +++ return responseBuilder.build(); ++ }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); +++ } +++ +++ protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) { +++ ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage(); +++ AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder() +++ .setMessageId(handleMessage.getMessageId()) +++ .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle()); +++ if (batchAckResult.getProxyException() != null) { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException())); +++ } else { +++ AckResult ackResult = batchAckResult.getAckResult(); +++ if (AckStatus.OK.equals(ackResult.getStatus())) { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); +++ } else { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")); +++ } ++ } ++- return future; +++ return resultBuilder.build(); ++ } ++ ++- protected CompletableFuture processAckMessage(ProxyContext ctx, AckMessageRequest request, +++ protected CompletableFuture ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) { +++ CompletableFuture resultFuture = new CompletableFuture<>(); +++ CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; +++ for (int i = 0; i < request.getEntriesCount(); i++) { +++ futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i)); +++ } +++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +++ if (throwable != null) { +++ resultFuture.completeExceptionally(throwable); +++ return; +++ } +++ +++ Set responseCodes = new HashSet<>(); +++ List entryList = new ArrayList<>(); +++ for (CompletableFuture entryFuture : futures) { +++ AckMessageResultEntry entryResult = entryFuture.join(); +++ responseCodes.add(entryResult.getStatus().getCode()); +++ entryList.add(entryResult); +++ } +++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() +++ .addAllEntries(entryList); +++ setAckResponseStatus(responseBuilder, responseCodes); +++ resultFuture.complete(responseBuilder.build()); +++ }); +++ return resultFuture; +++ } +++ +++ protected CompletableFuture processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request, ++ AckMessageEntry ackMessageEntry) { ++ CompletableFuture future = new CompletableFuture<>(); ++ ++ try { ++- String handleString = ackMessageEntry.getReceiptHandle(); ++- ++- String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); ++- MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); ++- if (messageReceiptHandle != null) { ++- handleString = messageReceiptHandle.getReceiptHandleStr(); ++- } +++ String handleString = this.getHandleString(ctx, group, request, ackMessageEntry); ++ CompletableFuture ackResultFuture = this.messagingProcessor.ackMessage( ++ ctx, ++ ReceiptHandle.decode(handleString), ++ ackMessageEntry.getMessageId(), ++ group, ++- GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); +++ topic +++ ); ++ ackResultFuture.thenAccept(result -> { ++ future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result)); ++ }).exceptionally(t -> { ++@@ -139,4 +180,25 @@ public class AckMessageActivity extends AbstractMessingActivity { ++ .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")) ++ .build(); ++ } +++ +++ protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set responseCodes) { +++ if (responseCodes.size() > 1) { +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); +++ } else if (responseCodes.size() == 1) { +++ Code code = responseCodes.stream().findAny().get(); +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); +++ } else { +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); +++ } +++ } +++ +++ protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) { +++ String handleString = ackMessageEntry.getReceiptHandle(); +++ +++ MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); +++ if (messageReceiptHandle != null) { +++ handleString = messageReceiptHandle.getReceiptHandleStr(); +++ } +++ return handleString; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++index b61c3df9e..c63212c23 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++@@ -27,6 +27,8 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { ++ protected MessagingProcessor messagingProcessor; ++ protected ServiceManager serviceManager; ++ +++ protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); +++ ++ public AbstractProcessor(MessagingProcessor messagingProcessor, ++ ServiceManager serviceManager) { ++ this.messagingProcessor = messagingProcessor; ++@@ -35,7 +37,7 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { ++ ++ protected void validateReceiptHandle(ReceiptHandle handle) { ++ if (handle.isExpired()) { ++- throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); +++ throw EXPIRED_HANDLE_PROXY_EXCEPTION; ++ } ++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++new file mode 100644 ++index 000000000..dfb9c9b9e ++--- /dev/null +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++@@ -0,0 +1,53 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.proxy.processor; +++ +++import org.apache.rocketmq.client.consumer.AckResult; +++import org.apache.rocketmq.proxy.common.ProxyException; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; +++ +++public class BatchAckResult { +++ +++ private final ReceiptHandleMessage receiptHandleMessage; +++ private AckResult ackResult; +++ private ProxyException proxyException; +++ +++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, +++ AckResult ackResult) { +++ this.receiptHandleMessage = receiptHandleMessage; +++ this.ackResult = ackResult; +++ } +++ +++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, +++ ProxyException proxyException) { +++ this.receiptHandleMessage = receiptHandleMessage; +++ this.proxyException = proxyException; +++ } +++ +++ public ReceiptHandleMessage getReceiptHandleMessage() { +++ return receiptHandleMessage; +++ } +++ +++ public AckResult getAckResult() { +++ return ackResult; +++ } +++ +++ public ProxyException getProxyException() { +++ return proxyException; +++ } +++} ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++index 656a6339d..f3522b374 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++@@ -48,6 +48,7 @@ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.common.utils.ProxyUtils; ++ import org.apache.rocketmq.proxy.service.ServiceManager; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; ++@@ -241,6 +242,69 @@ public class ConsumerProcessor extends AbstractProcessor { ++ return FutureUtils.addExecutor(future, this.executor); ++ } ++ +++ public CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ) { +++ CompletableFuture> future = new CompletableFuture<>(); +++ try { +++ List batchAckResultList = new ArrayList<>(handleMessageList.size()); +++ Map> brokerHandleListMap = new HashMap<>(); +++ +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ if (handleMessage.getReceiptHandle().isExpired()) { +++ batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION)); +++ continue; +++ } +++ List brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>()); +++ brokerHandleList.add(handleMessage); +++ } +++ +++ if (brokerHandleListMap.isEmpty()) { +++ return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor); +++ } +++ Set>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet(); +++ CompletableFuture>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()]; +++ int futureIndex = 0; +++ for (Map.Entry> entry : brokerHandleListMapEntrySet) { +++ futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis); +++ } +++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +++ if (throwable != null) { +++ future.completeExceptionally(throwable); +++ } +++ for (CompletableFuture> resultFuture : futures) { +++ batchAckResultList.addAll(resultFuture.join()); +++ } +++ future.complete(batchAckResultList); +++ }); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return FutureUtils.addExecutor(future, this.executor); +++ } +++ +++ protected CompletableFuture> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List handleMessageList, long timeoutMillis) { +++ return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis) +++ .thenApply(result -> { +++ List results = new ArrayList<>(); +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ results.add(new BatchAckResult(handleMessage, result)); +++ } +++ return results; +++ }) +++ .exceptionally(throwable -> { +++ List results = new ArrayList<>(); +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable))); +++ } +++ return results; +++ }); +++ } +++ ++ public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, ++ String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++index 188cb7b9b..ba150051b 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++@@ -46,6 +46,7 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.service.ServiceManager; ++ import org.apache.rocketmq.proxy.service.ServiceManagerFactory; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; ++@@ -183,6 +184,12 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen ++ return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis); ++ } ++ +++ @Override +++ public CompletableFuture> batchAckMessage(ProxyContext ctx, +++ List handleMessageList, String consumerGroup, String topic, long timeoutMillis) { +++ return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis); +++ } +++ ++ @Override ++ public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, ++ String groupName, String topicName, long invisibleTime, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++index d86be0bd8..2ae7418ba 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++@@ -37,6 +37,7 @@ import org.apache.rocketmq.proxy.common.Address; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; ++@@ -155,6 +156,23 @@ public interface MessagingProcessor extends StartAndShutdown { ++ long timeoutMillis ++ ); ++ +++ default CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic +++ ) { +++ return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS); +++ } +++ +++ CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ); +++ ++ default CompletableFuture changeInvisibleTime( ++ ProxyContext ctx, ++ ReceiptHandle handle, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++index 9f163f1b9..70b72deae 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++@@ -20,9 +20,11 @@ import com.google.common.collect.Lists; ++ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; +++import java.util.stream.Collectors; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.PopResult; ++ import org.apache.rocketmq.client.consumer.PullResult; +++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ import org.apache.rocketmq.common.message.Message; ++@@ -31,7 +33,6 @@ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.TopicRouteService; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -137,6 +138,19 @@ public class ClusterMessageService implements MessageService { ++ ); ++ } ++ +++ @Override +++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, String consumerGroup, +++ String topic, long timeoutMillis) { +++ List extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList()); +++ return this.mqClientAPIFactory.getClient().batchAckMessageAsync( +++ this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()), +++ topic, +++ consumerGroup, +++ extraInfoList, +++ timeoutMillis +++ ); +++ } +++ ++ @Override ++ public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, ++ PullMessageRequestHeader requestHeader, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++index eb2c4d9ee..ca7dcc9eb 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.service.message; ++ import io.netty.channel.ChannelHandlerContext; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; +++import java.util.BitSet; ++ import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.List; ++@@ -54,6 +55,8 @@ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; +++import org.apache.rocketmq.remoting.protocol.body.BatchAck; +++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; ++@@ -364,6 +367,61 @@ public class LocalMessageService implements MessageService { ++ }); ++ } ++ +++ @Override +++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, +++ String consumerGroup, String topic, long timeoutMillis) { +++ SimpleChannel channel = channelManager.createChannel(ctx); +++ ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); +++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); +++ +++ Map batchAckMap = new HashMap<>(); +++ for (ReceiptHandleMessage receiptHandleMessage : handleList) { +++ String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle(); +++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo); +++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + +++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" + +++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + +++ ExtraInfoUtil.getPopTime(extraInfoData); +++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { +++ BatchAck newBatchAck = new BatchAck(); +++ newBatchAck.setConsumerGroup(consumerGroup); +++ newBatchAck.setTopic(topic); +++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); +++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); +++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); +++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); +++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); +++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); +++ newBatchAck.setBitSet(new BitSet()); +++ return newBatchAck; +++ }); +++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); +++ } +++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); +++ requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName()); +++ requestBody.setAcks(new ArrayList<>(batchAckMap.values())); +++ +++ command.setBody(requestBody.encode()); +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ RemotingCommand response = brokerController.getAckMessageProcessor() +++ .processRequest(channelHandlerContext, command); +++ future.complete(response); +++ } catch (Exception e) { +++ log.error("Fail to process batchAckMessage command", e); +++ future.completeExceptionally(e); +++ } +++ return future.thenApply(r -> { +++ AckResult ackResult = new AckResult(); +++ if (ResponseCode.SUCCESS == r.getCode()) { +++ ackResult.setStatus(AckStatus.OK); +++ } else { +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ } +++ return ackResult; +++ }); +++ } +++ ++ @Override ++ public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, ++ PullMessageRequestHeader requestHeader, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++index 15da17154..58a835adb 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++@@ -91,6 +91,14 @@ public interface MessageService { ++ long timeoutMillis ++ ); ++ +++ CompletableFuture batchAckMessage( +++ ProxyContext ctx, +++ List handleList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ); +++ ++ CompletableFuture pullMessage( ++ ProxyContext ctx, ++ AddressableMessageQueue messageQueue, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++new file mode 100644 ++index 000000000..ae63fed49 ++--- /dev/null +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++@@ -0,0 +1,39 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.proxy.service.message; +++ +++import org.apache.rocketmq.common.consumer.ReceiptHandle; +++ +++public class ReceiptHandleMessage { +++ +++ private final ReceiptHandle receiptHandle; +++ private final String messageId; +++ +++ public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) { +++ this.receiptHandle = receiptHandle; +++ this.messageId = messageId; +++ } +++ +++ public ReceiptHandle getReceiptHandle() { +++ return receiptHandle; +++ } +++ +++ public String getMessageId() { +++ return messageId; +++ } +++} ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++index 49fdfc6a8..3c4746105 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++@@ -20,21 +20,32 @@ package org.apache.rocketmq.proxy.grpc.v2.consumer; ++ import apache.rocketmq.v2.AckMessageEntry; ++ import apache.rocketmq.v2.AckMessageRequest; ++ import apache.rocketmq.v2.AckMessageResponse; +++import apache.rocketmq.v2.AckMessageResultEntry; ++ import apache.rocketmq.v2.Code; ++ import apache.rocketmq.v2.Resource; +++import java.util.ArrayList; +++import java.util.HashMap; +++import java.util.List; +++import java.util.Map; ++ import java.util.concurrent.CompletableFuture; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.AckStatus; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +++import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +++import org.apache.rocketmq.proxy.processor.BatchAckResult; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.junit.Before; ++ import org.junit.Test; +++import org.mockito.stubbing.Answer; ++ ++ import static org.junit.Assert.assertEquals; ++ import static org.mockito.ArgumentMatchers.any; +++import static org.mockito.ArgumentMatchers.anyList; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; +++import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.when; ++ ++ public class AckMessageActivityTest extends BaseActivityTest { ++@@ -52,43 +63,197 @@ public class AckMessageActivityTest extends BaseActivityTest { ++ ++ @Test ++ public void testAckMessage() throws Throwable { ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString())) +++ ConfigurationManager.getProxyConfig().setEnableBatchAck(false); +++ +++ String msg1 = "msg1"; +++ String msg2 = "msg2"; +++ String msg3 = "msg3"; +++ +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString())) ++ .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired")); ++ ++ AckResult msg2AckResult = new AckResult(); ++ msg2AckResult.setStatus(AckStatus.OK); ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString())) +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString())) ++ .thenReturn(CompletableFuture.completedFuture(msg2AckResult)); ++ ++ AckResult msg3AckResult = new AckResult(); ++ msg3AckResult.setStatus(AckStatus.NO_EXIST); ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString())) +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString())) ++ .thenReturn(CompletableFuture.completedFuture(msg3AckResult)); ++ ++- AckMessageResponse response = this.ackMessageActivity.ackMessage( ++- createContext(), ++- AckMessageRequest.newBuilder() ++- .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++- .setGroup(Resource.newBuilder().setName(GROUP).build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg1") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++- .build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg2") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++- .build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg3") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++- .build()) ++- .build() ++- ).get(); ++- ++- assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); ++- assertEquals(3, response.getEntriesCount()); ++- assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); ++- assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); ++- assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg1) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg2) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.OK, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg3) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg1) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg2) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg3) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ +++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +++ assertEquals(3, response.getEntriesCount()); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); +++ assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); +++ } +++ } +++ +++ @Test +++ public void testAckMessageInBatch() throws Throwable { +++ ConfigurationManager.getProxyConfig().setEnableBatchAck(true); +++ +++ String successMessageId = "msg1"; +++ String notOkMessageId = "msg2"; +++ String exceptionMessageId = "msg3"; +++ +++ doAnswer((Answer>>) invocation -> { +++ List receiptHandleMessageList = invocation.getArgument(1, List.class); +++ List batchAckResultList = new ArrayList<>(); +++ for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) { +++ BatchAckResult batchAckResult; +++ if (receiptHandleMessage.getMessageId().equals(successMessageId)) { +++ AckResult ackResult = new AckResult(); +++ ackResult.setStatus(AckStatus.OK); +++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); +++ } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) { +++ AckResult ackResult = new AckResult(); +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); +++ } else { +++ batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "")); +++ } +++ batchAckResultList.add(batchAckResult); +++ } +++ return CompletableFuture.completedFuture(batchAckResultList); +++ }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString()); +++ +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(successMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.OK, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(notOkMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(exceptionMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(successMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(notOkMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(exceptionMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ +++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +++ assertEquals(3, response.getEntriesCount()); +++ Map msgCode = new HashMap<>(); +++ for (AckMessageResultEntry entry : response.getEntriesList()) { +++ msgCode.put(entry.getMessageId(), entry.getStatus().getCode()); +++ } +++ assertEquals(Code.OK, msgCode.get(successMessageId)); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId)); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId)); +++ } ++ } ++ } ++\ No newline at end of file ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++index 5c1ea9627..072630e39 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++@@ -66,14 +66,6 @@ public class BaseProcessorTest extends InitConfigTest { ++ protected ProxyRelayService proxyRelayService; ++ @Mock ++ protected MetadataService metadataService; ++- @Mock ++- protected ProducerProcessor producerProcessor; ++- @Mock ++- protected ConsumerProcessor consumerProcessor; ++- @Mock ++- protected TransactionProcessor transactionProcessor; ++- @Mock ++- protected ClientProcessor clientProcessor; ++ ++ public void before() throws Throwable { ++ super.before(); ++@@ -92,6 +84,13 @@ public class BaseProcessorTest extends InitConfigTest { ++ } ++ ++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) { +++ return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(), +++ RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), +++ RANDOM.nextInt(Integer.MAX_VALUE), "mockBroker"); +++ } +++ +++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime, +++ long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) { ++ MessageExt messageExt = new MessageExt(); ++ messageExt.setTopic(topic); ++ messageExt.setTags(tags); ++@@ -100,8 +99,7 @@ public class BaseProcessorTest extends InitConfigTest { ++ messageExt.setMsgId(MessageClientIDSetter.createUniqID()); ++ messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE)); ++ MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, ++- ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime, ++- RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE))); +++ ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset)); ++ return messageExt; ++ } ++ ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++index 717e86fc0..db268a06e 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++@@ -20,8 +20,11 @@ package org.apache.rocketmq.proxy.processor; ++ import com.google.common.collect.Sets; ++ import java.time.Duration; ++ import java.util.ArrayList; +++import java.util.Collections; +++import java.util.HashMap; ++ import java.util.HashSet; ++ import java.util.List; +++import java.util.Map; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.Executors; ++@@ -39,7 +42,10 @@ import org.apache.rocketmq.common.message.MessageClientIDSetter; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.proxy.common.ProxyContext; +++import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +++import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.common.utils.ProxyUtils; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.MessageQueueView; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -50,16 +56,22 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; ++ import org.junit.Before; ++ import org.junit.Test; ++ import org.mockito.ArgumentCaptor; +++import org.mockito.stubbing.Answer; ++ ++ import static org.assertj.core.api.Assertions.assertThat; ++ import static org.junit.Assert.assertEquals; ++ import static org.junit.Assert.assertNotNull; +++import static org.junit.Assert.assertNull; ++ import static org.junit.Assert.assertSame; ++ import static org.mockito.ArgumentMatchers.any; +++import static org.mockito.ArgumentMatchers.anyList; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; +++import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.mock; +++import static org.mockito.Mockito.never; +++import static org.mockito.Mockito.verify; ++ import static org.mockito.Mockito.when; ++ ++ public class ConsumerProcessorTest extends BaseProcessorTest { ++@@ -162,6 +174,109 @@ public class ConsumerProcessorTest extends BaseProcessorTest { ++ assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo()); ++ } ++ +++ @Test +++ public void testBatchAckExpireMessage() throws Throwable { +++ String brokerName1 = "brokerName1"; +++ +++ List receiptHandleMessageList = new ArrayList<>(); +++ for (int i = 0; i < 3; i++) { +++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, +++ 0, 0, 0, i, brokerName1); +++ ReceiptHandle expireHandle = create(expireMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); +++ } +++ +++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); +++ +++ verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); +++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ assertNull(batchAckResult.getAckResult()); +++ assertNotNull(batchAckResult.getProxyException()); +++ assertNotNull(batchAckResult.getReceiptHandleMessage()); +++ } +++ +++ } +++ +++ @Test +++ public void testBatchAckMessage() throws Throwable { +++ String brokerName1 = "brokerName1"; +++ String brokerName2 = "brokerName2"; +++ String errThrowBrokerName = "errThrowBrokerName"; +++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, +++ 0, 0, 0, 0, brokerName1); +++ ReceiptHandle expireHandle = create(expireMessage); +++ +++ List receiptHandleMessageList = new ArrayList<>(); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); +++ List broker1Msg = new ArrayList<>(); +++ List broker2Msg = new ArrayList<>(); +++ +++ long now = System.currentTimeMillis(); +++ int msgNum = 3; +++ for (int i = 0; i < msgNum; i++) { +++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, i + 1, brokerName1); +++ ReceiptHandle brokerHandle = create(brokerMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); +++ broker1Msg.add(brokerMessage.getMsgId()); +++ } +++ for (int i = 0; i < msgNum; i++) { +++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, i + 1, brokerName2); +++ ReceiptHandle brokerHandle = create(brokerMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); +++ broker2Msg.add(brokerMessage.getMsgId()); +++ } +++ +++ // for this message, will throw exception in batchAckMessage +++ MessageExt errThrowMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, 0, errThrowBrokerName); +++ ReceiptHandle errThrowHandle = create(errThrowMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId())); +++ +++ Collections.shuffle(receiptHandleMessageList); +++ +++ doAnswer((Answer>) invocation -> { +++ List handleMessageList = invocation.getArgument(1, List.class); +++ AckResult ackResult = new AckResult(); +++ String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName(); +++ if (brokerName.equals(brokerName1)) { +++ ackResult.setStatus(AckStatus.OK); +++ } else if (brokerName.equals(brokerName2)) { +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ } else { +++ return FutureUtils.completeExceptionally(new RuntimeException()); +++ } +++ +++ return CompletableFuture.completedFuture(ackResult); +++ }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); +++ +++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); +++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); +++ +++ // check ackResult for each msg +++ Map msgBatchAckResult = new HashMap<>(); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult); +++ } +++ for (String msgId : broker1Msg) { +++ assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus()); +++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); +++ } +++ for (String msgId : broker2Msg) { +++ assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus()); +++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); +++ } +++ assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException()); +++ assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode()); +++ assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult()); +++ +++ assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException()); +++ assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode()); +++ assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult()); +++ } +++ ++ @Test ++ public void testChangeInvisibleTime() throws Throwable { ++ ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000)); ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++index 77a119a29..3f3a4ae40 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++@@ -220,6 +220,18 @@ public class MQClientAPIExtTest { ++ assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get()); ++ } ++ +++ @Test +++ public void testBatchAckMessageAsync() throws Exception { +++ AckResult ackResult = new AckResult(); +++ doAnswer((Answer) mock -> { +++ AckCallback ackCallback = mock.getArgument(2); +++ ackCallback.onSuccess(ackResult); +++ return null; +++ }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any()); +++ +++ assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get()); +++ } +++ ++ @Test ++ public void testChangeInvisibleTimeAsync() throws Exception { ++ AckResult ackResult = new AckResult(); ++-- ++2.32.0.windows.2 ++ +diff --git a/patch012-backport-enhance-rockdbconfigtojson.patch b/patch012-backport-enhance-rockdbconfigtojson.patch +new file mode 100644 +index 000000000..cf6128b24 +--- /dev/null ++++ b/patch012-backport-enhance-rockdbconfigtojson.patch +@@ -0,0 +1,2920 @@ ++From fec141481496c53a0db398367006c34264662d18 Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Wed, 23 Aug 2023 08:22:34 +0800 ++Subject: [PATCH 1/8] [ISSUE #7166] Optimize the display format of admin ++ (#7210) ++ ++--- ++ .../java/org/apache/rocketmq/tools/command/MQAdminStartup.java | 2 +- ++ .../command/acl/ClusterAclConfigVersionListSubCommand.java | 2 +- ++ .../tools/command/acl/DeleteAccessConfigSubCommand.java | 2 +- ++ .../rocketmq/tools/command/acl/GetAccessConfigSubCommand.java | 2 +- ++ .../tools/command/acl/UpdateAccessConfigSubCommand.java | 2 +- ++ .../tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java | 2 +- ++ .../tools/command/broker/BrokerConsumeStatsSubCommad.java | 2 +- ++ .../rocketmq/tools/command/broker/BrokerStatusSubCommand.java | 2 +- ++ .../tools/command/broker/CommitLogSetReadAheadSubCommand.java | 2 +- ++ .../tools/command/broker/DeleteExpiredCommitLogSubCommand.java | 2 +- ++ .../rocketmq/tools/command/broker/GetBrokerConfigCommand.java | 2 +- ++ .../rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java | 2 +- ++ .../tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java | 2 +- ++ .../broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java | 2 +- ++ .../tools/command/broker/ResetMasterFlushOffsetSubCommand.java | 2 +- ++ .../tools/command/broker/UpdateBrokerConfigSubCommand.java | 2 +- ++ .../broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java | 2 +- ++ .../rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java | 2 +- ++ .../rocketmq/tools/command/cluster/ClusterListSubCommand.java | 2 +- ++ .../tools/command/connection/ConsumerConnectionSubCommand.java | 2 +- ++ .../tools/command/connection/ProducerConnectionSubCommand.java | 2 +- ++ .../tools/command/consumer/ConsumerStatusSubCommand.java | 2 +- ++ .../tools/command/consumer/GetConsumerConfigSubCommand.java | 2 +- ++ .../tools/command/consumer/StartMonitoringSubCommand.java | 2 +- ++ .../tools/command/consumer/UpdateSubGroupSubCommand.java | 2 +- ++ .../rocketmq/tools/command/container/AddBrokerSubCommand.java | 2 +- ++ .../tools/command/container/RemoveBrokerSubCommand.java | 2 +- ++ .../command/controller/CleanControllerBrokerMetaSubCommand.java | 2 +- ++ .../command/controller/GetControllerMetaDataSubCommand.java | 2 +- ++ .../tools/command/controller/ReElectMasterSubCommand.java | 2 +- ++ .../rocketmq/tools/command/export/ExportConfigsCommand.java | 2 +- ++ .../rocketmq/tools/command/export/ExportMetadataCommand.java | 2 +- ++ .../rocketmq/tools/command/export/ExportMetricsCommand.java | 2 +- ++ .../rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java | 2 +- ++ .../apache/rocketmq/tools/command/ha/HAStatusSubCommand.java | 2 +- ++ .../rocketmq/tools/command/message/CheckMsgSendRTCommand.java | 2 +- ++ .../rocketmq/tools/command/message/ConsumeMessageCommand.java | 2 +- ++ .../tools/command/message/DumpCompactionLogCommand.java | 2 +- ++ .../tools/command/message/PrintMessageByQueueCommand.java | 2 +- ++ .../rocketmq/tools/command/message/PrintMessageSubCommand.java | 2 +- ++ .../rocketmq/tools/command/message/QueryMsgByIdSubCommand.java | 2 +- ++ .../rocketmq/tools/command/message/QueryMsgByKeySubCommand.java | 2 +- ++ .../tools/command/message/QueryMsgByOffsetSubCommand.java | 2 +- ++ .../tools/command/message/QueryMsgByUniqueKeySubCommand.java | 2 +- ++ .../tools/command/message/QueryMsgTraceByIdSubCommand.java | 2 +- ++ .../rocketmq/tools/command/message/SendMessageCommand.java | 2 +- ++ .../rocketmq/tools/command/namesrv/AddWritePermSubCommand.java | 2 +- ++ .../rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java | 2 +- ++ .../tools/command/offset/SkipAccumulationSubCommand.java | 2 +- ++ .../apache/rocketmq/tools/command/stats/StatsAllSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/AllocateMQSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/TopicClusterSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/TopicListSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/TopicRouteSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/TopicStatusSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/UpdateOrderConfCommand.java | 2 +- ++ .../tools/command/topic/UpdateStaticTopicSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/UpdateTopicSubCommand.java | 2 +- ++ 59 files changed, 59 insertions(+), 59 deletions(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++index 0c2618e91..890125ca0 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++@@ -278,7 +278,7 @@ public class MQAdminStartup { ++ System.out.printf("The most commonly used mqadmin commands are:%n"); ++ ++ for (SubCommand cmd : SUB_COMMANDS) { ++- System.out.printf(" %-25s %s%n", cmd.commandName(), cmd.commandDesc()); +++ System.out.printf(" %-35s %s%n", cmd.commandName(), cmd.commandDesc()); ++ } ++ ++ System.out.printf("%nSee 'mqadmin help ' for more information on a specific command.%n"); ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java ++index f8a00b1e0..26ed028fb 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java ++@@ -47,7 +47,7 @@ public class ClusterAclConfigVersionListSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "List all of acl config version information in cluster"; +++ return "List all of acl config version information in cluster."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java ++index fd3a92fff..a7f3d295a 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java ++@@ -42,7 +42,7 @@ public class DeleteAccessConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Delete Acl Config Account in broker"; +++ return "Delete Acl Config Account in broker."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java ++index 25844d6a1..f1c9a1496 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java ++@@ -49,7 +49,7 @@ public class GetAccessConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "List all of acl config information in cluster"; +++ return "List all of acl config information in cluster."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java ++index 3be40daa1..d8a06f92d 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java ++@@ -40,7 +40,7 @@ public class UpdateAccessConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update acl config yaml file in broker"; +++ return "Update acl config yaml file in broker."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java ++index ff662b506..9dacf1fae 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java ++@@ -37,7 +37,7 @@ public class UpdateGlobalWhiteAddrSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update global white address for acl Config File in broker"; +++ return "Update global white address for acl Config File in broker."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java ++index 3f2f90673..7658a2139 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java ++@@ -61,7 +61,7 @@ public class BrokerConsumeStatsSubCommad implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch broker consume stats data"; +++ return "Fetch broker consume stats data."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java ++index 830ff3425..ce934f547 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java ++@@ -44,7 +44,7 @@ public class BrokerStatusSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch broker runtime status data"; +++ return "Fetch broker runtime status data."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java ++index b00c7f5f5..4fdabfdf8 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java ++@@ -44,7 +44,7 @@ public class CommitLogSetReadAheadSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "set read ahead mode for all commitlog files"; +++ return "Set read ahead mode for all commitlog files."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java ++index a4b2a51ad..142bb7b3c 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java ++@@ -37,7 +37,7 @@ public class DeleteExpiredCommitLogSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Delete expired CommitLog files"; +++ return "Delete expired CommitLog files."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java ++index 5d86c10e4..c4762a296 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java ++@@ -45,7 +45,7 @@ public class GetBrokerConfigCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Get broker config by cluster or special broker"; +++ return "Get broker config by cluster or special broker."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java ++index abe8fc622..1a8961e04 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java ++@@ -38,7 +38,7 @@ public class GetBrokerEpochSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch broker epoch entries"; +++ return "Fetch broker epoch entries."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java ++index 7c54e650c..34b3ba7d3 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java ++@@ -47,7 +47,7 @@ public class GetColdDataFlowCtrInfoSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "get cold data flow ctr info"; +++ return "Get cold data flow ctr info."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java ++index b0477924f..f20407480 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java ++@@ -36,7 +36,7 @@ public class RemoveColdDataFlowCtrGroupConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "remove consumer from cold ctr config"; +++ return "Remove consumer from cold ctr config."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java ++index b2ac48c84..90451b51f 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java ++@@ -33,7 +33,7 @@ public class ResetMasterFlushOffsetSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Reset master flush offset in slave"; +++ return "Reset master flush offset in slave."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java ++index 98abeb6ae..62816ef03 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java ++@@ -37,7 +37,7 @@ public class UpdateBrokerConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update broker's config"; +++ return "Update broker's config."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java ++index d06a24b57..8d1a00077 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java ++@@ -39,7 +39,7 @@ public class UpdateColdDataFlowCtrGroupConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "addOrUpdate cold data flow ctr group config"; +++ return "Add or update cold data flow ctr group config."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java ++index 7253970bd..d755e9e5d 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java ++@@ -48,7 +48,7 @@ public class CLusterSendMsgRTCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "List All clusters Message Send RT"; +++ return "List All clusters Message Send RT."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java ++index a7a840a44..ede0fa5cf 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java ++@@ -41,7 +41,7 @@ public class ClusterListSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "List cluster infos"; +++ return "List cluster infos."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java ++index 630961e31..35f73d8a0 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java ++@@ -39,7 +39,7 @@ public class ConsumerConnectionSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query consumer's socket connection, client version and subscription"; +++ return "Query consumer's socket connection, client version and subscription."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java ++index 2533982c8..bde674ab2 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java ++@@ -36,7 +36,7 @@ public class ProducerConnectionSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query producer's socket connection and client version"; +++ return "Query producer's socket connection and client version."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java ++index 72b9c975e..d8f6f9aa9 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java ++@@ -47,7 +47,7 @@ public class ConsumerStatusSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query consumer's internal data structure"; +++ return "Query consumer's internal data structure."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java ++index 6095e7668..4a8253a02 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java ++@@ -43,7 +43,7 @@ public class GetConsumerConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Get consumer config by subscription group name"; +++ return "Get consumer config by subscription group name."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java ++index 2d08d0bd0..f5e140433 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java ++@@ -34,7 +34,7 @@ public class StartMonitoringSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Start Monitoring"; +++ return "Start Monitoring."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java ++index f87bafc93..b17da4de4 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java ++@@ -41,7 +41,7 @@ public class UpdateSubGroupSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update or create subscription group"; +++ return "Update or create subscription group."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java ++index e9e5be4a5..007d42ae6 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java ++@@ -33,7 +33,7 @@ public class AddBrokerSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Add a broker to specified container"; +++ return "Add a broker to specified container."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java ++index 7c455f858..ab25d8ebe 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java ++@@ -33,7 +33,7 @@ public class RemoveBrokerSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Remove a broker from specified container"; +++ return "Remove a broker from specified container."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java ++index 856e4b426..24ed02566 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java ++@@ -37,7 +37,7 @@ public class CleanControllerBrokerMetaSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Clean metadata of broker on controller"; +++ return "Clean metadata of broker on controller."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java ++index 70bd7f8e9..966443127 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java ++@@ -34,7 +34,7 @@ public class GetControllerMetaDataSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Get controller cluster's metadata"; +++ return "Get controller cluster's metadata."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java ++index 1affe81f9..a522a903d 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java ++@@ -37,7 +37,7 @@ public class ReElectMasterSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Re-elect the specified broker as master"; +++ return "Re-elect the specified broker as master."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++index b8191296d..03613b29c 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++@@ -42,7 +42,7 @@ public class ExportConfigsCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Export configs"; +++ return "Export configs."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java ++index 1f9cf7d96..748f7b16e 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java ++@@ -46,7 +46,7 @@ public class ExportMetadataCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Export metadata"; +++ return "Export metadata."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java ++index a793b4b84..5d8bb37ba 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java ++@@ -56,7 +56,7 @@ public class ExportMetricsCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Export metrics"; +++ return "Export metrics."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java ++index 44b3ec3e1..b6231e4f9 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java ++@@ -40,7 +40,7 @@ public class GetSyncStateSetSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch syncStateSet for target brokers"; +++ return "Fetch syncStateSet for target brokers."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java ++index b1795e046..931658a08 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java ++@@ -41,7 +41,7 @@ public class HAStatusSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch ha runtime status data"; +++ return "Fetch ha runtime status data."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java ++index 4c6d5ffb6..b15b59d50 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java ++@@ -40,7 +40,7 @@ public class CheckMsgSendRTCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Check message send response time"; +++ return "Check message send response time."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java ++index 8aed59ea4..02ff53269 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java ++@@ -70,7 +70,7 @@ public class ConsumeMessageCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Consume message"; +++ return "Consume message."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java ++index ae6d9bdcf..eee8f3d4b 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java ++@@ -38,7 +38,7 @@ import java.nio.file.Paths; ++ public class DumpCompactionLogCommand implements SubCommand { ++ @Override ++ public String commandDesc() { ++- return "parse compaction log to message"; +++ return "Parse compaction log to message."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java ++index 654560167..0418e88a7 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java ++@@ -108,7 +108,7 @@ public class PrintMessageByQueueCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Print Message Detail"; +++ return "Print Message Detail by queueId."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java ++index d01c36d42..bb82f5079 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java ++@@ -62,7 +62,7 @@ public class PrintMessageSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Print Message Detail"; +++ return "Print Message Detail."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java ++index 2880477f1..b42612150 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java ++@@ -186,7 +186,7 @@ public class QueryMsgByIdSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query Message by Id"; +++ return "Query Message by Id."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java ++index ba7b00c3b..64627fd19 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java ++@@ -36,7 +36,7 @@ public class QueryMsgByKeySubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query Message by Key"; +++ return "Query Message by Key."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java ++index d27313af1..14d0625fd 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java ++@@ -39,7 +39,7 @@ public class QueryMsgByOffsetSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query Message by offset"; +++ return "Query Message by offset."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java ++index 1b28f8be1..b71cee901 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java ++@@ -141,7 +141,7 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query Message by Unique key"; +++ return "Query Message by Unique key."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java ++index 2b982efef..2c546ec56 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java ++@@ -65,7 +65,7 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query a message trace"; +++ return "Query a message trace."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java ++index 836ee192b..970da6b16 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java ++@@ -41,7 +41,7 @@ public class SendMessageCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Send a message"; +++ return "Send a message."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java ++index 98542d065..0b0a075bd 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java ++@@ -34,7 +34,7 @@ public class AddWritePermSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Add write perm of broker in all name server you defined in the -n param"; +++ return "Add write perm of broker in all name server you defined in the -n param."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java ++index 213931ed8..637dd52c8 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java ++@@ -34,7 +34,7 @@ public class WipeWritePermSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Wipe write perm of broker in all name server you defined in the -n param"; +++ return "Wipe write perm of broker in all name server you defined in the -n param."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java ++index 139821f9c..b22491a59 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java ++@@ -41,7 +41,7 @@ public class SkipAccumulationSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Skip all messages that are accumulated (not consumed) currently"; +++ return "Skip all messages that are accumulated (not consumed) currently."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java ++index 1d49bbe11..96097a93e 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java ++@@ -144,7 +144,7 @@ public class StatsAllSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Topic and Consumer tps stats"; +++ return "Topic and Consumer tps stats."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java ++index 3fa42f297..6a9b81eb8 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java ++@@ -41,7 +41,7 @@ public class AllocateMQSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Allocate MQ"; +++ return "Allocate MQ."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java ++index 1dab693d9..098f34ff0 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java ++@@ -34,7 +34,7 @@ public class TopicClusterSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Get cluster info for topic"; +++ return "Get cluster info for topic."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java ++index 346bac704..d9a279f80 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java ++@@ -45,7 +45,7 @@ public class TopicListSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch all topic list from name server"; +++ return "Fetch all topic list from name server."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java ++index f2dabec4e..70949d388 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java ++@@ -42,7 +42,7 @@ public class TopicRouteSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Examine topic route info"; +++ return "Examine topic route info."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java ++index fdb249fab..a1619eced 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java ++@@ -40,7 +40,7 @@ public class TopicStatusSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Examine topic Status info"; +++ return "Examine topic Status info."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java ++index bebc646b4..3040d04c2 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java ++@@ -36,7 +36,7 @@ public class UpdateOrderConfCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Create or update or delete order conf"; +++ return "Create or update or delete order conf."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java ++index 85a18c654..3daeee86c 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java ++@@ -48,7 +48,7 @@ public class UpdateStaticTopicSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update or create static topic, which has fixed number of queues"; +++ return "Update or create static topic, which has fixed number of queues."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java ++index aaa881538..d27cd1861 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java ++@@ -44,7 +44,7 @@ public class UpdateTopicPermSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update topic perm"; +++ return "Update topic perm."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java ++index b68463396..298914175 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java ++@@ -42,7 +42,7 @@ public class UpdateTopicSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update or create topic"; +++ return "Update or create topic."; ++ } ++ ++ @Override ++-- ++2.32.0.windows.2 ++ ++ ++From 744167bd01fab6821b4d5ae1794dc845153d5156 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Wed, 23 Aug 2023 08:32:17 +0800 ++Subject: [PATCH 2/8] [ISSUE #7142] Add command `RocksDBConfigToJson` to ++ inspect rocksdb content (#7180) ++ ++* feat: add command `RocksDBConfigToJson` to inspect rocksdb content ++ ++Signed-off-by: Ziy1-Tan ++ ++* refactor: fix style ++ ++--------- ++ ++Signed-off-by: Ziy1-Tan ++Co-authored-by: Ziy1-Tan ++--- ++ .../tools/command/MQAdminStartup.java | 2 + ++ .../metadata/RocksDBConfigToJsonCommand.java | 118 ++++++++++++++++++ ++ .../metadata/KvConfigToJsonCommandTest.java | 65 ++++++++++ ++ 3 files changed, 185 insertions(+) ++ create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++ create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++index 890125ca0..324aa1856 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++@@ -80,6 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; ++ import org.apache.rocketmq.tools.command.message.SendMessageCommand; +++import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; ++ import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; ++ import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; ++ import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; ++@@ -211,6 +212,7 @@ public class MQAdminStartup { ++ ++ initCommand(new ClusterListSubCommand()); ++ initCommand(new TopicListSubCommand()); +++ initCommand(new RocksDBConfigToJsonCommand()); ++ ++ initCommand(new UpdateKvConfigCommand()); ++ initCommand(new DeleteKvConfigCommand()); ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++new file mode 100644 ++index 000000000..3053f4684 ++--- /dev/null +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++@@ -0,0 +1,118 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.metadata; +++ +++import com.alibaba.fastjson.JSONObject; +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.Option; +++import org.apache.commons.cli.Options; +++import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.common.config.RocksDBConfigManager; +++import org.apache.rocketmq.common.utils.DataConverter; +++import org.apache.rocketmq.remoting.RPCHook; +++import org.apache.rocketmq.tools.command.SubCommand; +++import org.apache.rocketmq.tools.command.SubCommandException; +++ +++import java.io.File; +++import java.util.HashMap; +++import java.util.Map; +++ +++public class RocksDBConfigToJsonCommand implements SubCommand { +++ private static final String TOPICS_JSON_CONFIG = "topics"; +++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; +++ +++ @Override +++ public String commandName() { +++ return "rocksDBConfigToJson"; +++ } +++ +++ @Override +++ public String commandDesc() { +++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; +++ } +++ +++ @Override +++ public Options buildCommandlineOptions(Options options) { +++ Option pathOption = new Option("p", "path", true, +++ "Absolute path to the metadata directory"); +++ pathOption.setRequired(true); +++ options.addOption(pathOption); +++ +++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + +++ "topics/subscriptionGroups"); +++ configTypeOption.setRequired(true); +++ options.addOption(configTypeOption); +++ +++ return options; +++ } +++ +++ @Override +++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { +++ String path = commandLine.getOptionValue("path").trim(); +++ if (StringUtils.isEmpty(path) || !new File(path).exists()) { +++ System.out.print("Rocksdb path is invalid.\n"); +++ return; +++ } +++ +++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); +++ +++ final long memTableFlushInterval = 60 * 60 * 1000L; +++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); +++ try { +++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { +++ // for topics.json +++ final Map topicsJsonConfig = new HashMap<>(); +++ final Map topicConfigTable = new HashMap<>(); +++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { +++ final String topic = new String(key, DataConverter.charset); +++ final String topicConfig = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig); +++ topicConfigTable.put(topic, jsonObject); +++ }); +++ +++ if (isLoad) { +++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); +++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); +++ System.out.print(topicsJsonStr + "\n"); +++ return; +++ } +++ } +++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { +++ // for subscriptionGroup.json +++ final Map subscriptionGroupJsonConfig = new HashMap<>(); +++ final Map subscriptionGroupTable = new HashMap<>(); +++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { +++ final String subscriptionGroup = new String(key, DataConverter.charset); +++ final String subscriptionGroupConfig = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); +++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); +++ }); +++ +++ if (isLoad) { +++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", +++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); +++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); +++ System.out.print(subscriptionGroupJsonStr + "\n"); +++ return; +++ } +++ } +++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); +++ } finally { +++ kvConfigManager.stop(); +++ } +++ } +++} ++diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++new file mode 100644 ++index 000000000..b2f66c7b0 ++--- /dev/null +++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++@@ -0,0 +1,65 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.metadata; +++ +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.DefaultParser; +++import org.apache.commons.cli.Options; +++import org.apache.rocketmq.srvutil.ServerUtil; +++import org.apache.rocketmq.tools.command.SubCommandException; +++import org.junit.Test; +++ +++import java.io.File; +++ +++import static org.assertj.core.api.Assertions.assertThat; +++ +++public class KvConfigToJsonCommandTest { +++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; +++ +++ @Test +++ public void testExecute() throws SubCommandException { +++ { +++ String[] cases = new String[]{"topics", "subscriptionGroups"}; +++ for (String c : cases) { +++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ Options options = ServerUtil.buildCommandlineOptions(new Options()); +++ String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c}; +++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, +++ cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.execute(commandLine, options, null); +++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c); +++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c); +++ } +++ } +++ // invalid cases +++ { +++ String[][] cases = new String[][]{ +++ {"-p " + BASE_PATH + "tmpPath", "-t topics"}, +++ {"-p ", "-t topics"}, +++ {"-p " + BASE_PATH + "topics", "-t invalid_type"} +++ }; +++ +++ for (String[] c : cases) { +++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ Options options = ServerUtil.buildCommandlineOptions(new Options()); +++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, +++ cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.execute(commandLine, options, null); +++ } +++ } +++ } +++} ++-- ++2.32.0.windows.2 ++ ++ ++From bdede35db365a49b211cdc249c68b0f60a3df46d Mon Sep 17 00:00:00 2001 ++From: mxsm ++Date: Wed, 23 Aug 2023 08:34:56 +0800 ++Subject: [PATCH 3/8] [ISSUE #7124] Fix the typos in the code comments (#7125) ++ ++--- ++ .../apache/rocketmq/broker/processor/ReplyMessageProcessor.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++index b2db356c8..d3bb048f7 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++@@ -234,7 +234,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor { ++ } else { ++ response.setCode(ResponseCode.SUCCESS); ++ response.setRemark(null); ++- //set to zore to avoid client decoding exception +++ //set to zero to avoid client decoding exception ++ responseHeader.setMsgId("0"); ++ responseHeader.setQueueId(queueIdInt); ++ responseHeader.setQueueOffset(0L); ++-- ++2.32.0.windows.2 ++ ++ ++From 9bb73b9a38548b99ac5126c40380c3c2e7fc586e Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 23 Aug 2023 09:46:27 +0800 ++Subject: [PATCH 4/8] [#ISSUE 7222] Bug fix and refactoring of the Indexfile in ++ tiered storage (#7224) ++ ++--- ++ .../tieredstore/file/TieredIndexFile.java | 38 +++++++-- ++ .../tieredstore/file/TieredIndexFileTest.java | 84 +++++-------------- ++ 2 files changed, 52 insertions(+), 70 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++index 50beb01ae..eda5e0106 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ +++import com.google.common.annotations.VisibleForTesting; ++ import java.io.File; ++ import java.io.IOException; ++ import java.nio.ByteBuffer; ++@@ -99,7 +100,7 @@ public class TieredIndexFile { ++ this::doScheduleTask, 10, 10, TimeUnit.SECONDS); ++ } ++ ++- private void doScheduleTask() { +++ protected void doScheduleTask() { ++ try { ++ curFileLock.lock(); ++ try { ++@@ -145,6 +146,11 @@ public class TieredIndexFile { ++ } ++ } ++ +++ @VisibleForTesting +++ public MappedFile getPreMappedFile() { +++ return preMappedFile; +++ } +++ ++ private void initFile() throws IOException { ++ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); ++ initIndexFileHeader(curMappedFile); ++@@ -156,19 +162,26 @@ public class TieredIndexFile { ++ ++ if (isFileSealed(curMappedFile)) { ++ if (preFileExists) { ++- preFile.delete(); +++ if (preFile.delete()) { +++ logger.info("Pre IndexFile deleted success", preFilepath); +++ } else { +++ logger.error("Pre IndexFile deleted failed", preFilepath); +++ } ++ } ++ boolean rename = curMappedFile.renameTo(preFilepath); ++ if (rename) { ++ preMappedFile = curMappedFile; ++ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); +++ initIndexFileHeader(curMappedFile); ++ preFileExists = true; ++ } ++ } +++ ++ if (preFileExists) { ++ synchronized (TieredIndexFile.class) { ++ if (inflightCompactFuture.isDone()) { ++- inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(new CompactTask(storeConfig, preMappedFile, flatFile), null); +++ inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit( +++ new CompactTask(storeConfig, preMappedFile, flatFile), null); ++ } ++ } ++ } ++@@ -261,7 +274,8 @@ public class TieredIndexFile { ++ } ++ } ++ ++- public CompletableFuture>> queryAsync(String topic, String key, long beginTime, long endTime) { +++ public CompletableFuture>> queryAsync(String topic, String key, long beginTime, +++ long endTime) { ++ int hashCode = indexKeyHashMethod(buildKey(topic, key)); ++ int slotPosition = hashCode % maxHashSlotNum; ++ List fileSegmentList = flatFile.getFileListByTime(beginTime, endTime); ++@@ -355,7 +369,7 @@ public class TieredIndexFile { ++ private final int fileMaxSize; ++ private MappedFile originFile; ++ private TieredFlatFile fileQueue; ++- private final MappedFile compactFile; +++ private MappedFile compactFile; ++ ++ public CompactTask(TieredMessageStoreConfig storeConfig, MappedFile originFile, ++ TieredFlatFile fileQueue) throws IOException { ++@@ -381,6 +395,17 @@ public class TieredIndexFile { ++ } catch (Throwable throwable) { ++ logger.error("TieredIndexFile#compactTask: compact index file failed:", throwable); ++ } +++ +++ try { +++ if (originFile != null) { +++ originFile.destroy(-1); +++ } +++ if (compactFile != null) { +++ compactFile.destroy(-1); +++ } +++ } catch (Throwable throwable) { +++ logger.error("TieredIndexFile#compactTask: destroy index file failed:", throwable); +++ } ++ } ++ ++ public void compact() { ++@@ -396,6 +421,8 @@ public class TieredIndexFile { ++ fileQueue.commit(true); ++ compactFile.destroy(-1); ++ originFile.destroy(-1); +++ compactFile = null; +++ originFile = null; ++ } ++ ++ private void buildCompactFile() { ++@@ -414,6 +441,7 @@ public class TieredIndexFile { ++ if (slotValue != -1) { ++ int indexTotalSize = 0; ++ int indexPosition = slotValue; +++ ++ while (indexPosition >= 0 && indexPosition < maxIndexNum) { ++ int indexOffset = INDEX_FILE_HEADER_SIZE + maxHashSlotNum * INDEX_FILE_HASH_SLOT_SIZE ++ + indexPosition * INDEX_FILE_HASH_ORIGIN_INDEX_SIZE; ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++index 7ef49578d..262d6645b 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++@@ -19,9 +19,8 @@ package org.apache.rocketmq.tieredstore.file; ++ import com.sun.jna.Platform; ++ import java.io.IOException; ++ import java.nio.ByteBuffer; +++import java.time.Duration; ++ import java.util.List; ++-import java.util.concurrent.TimeUnit; ++-import org.apache.commons.lang3.SystemUtils; ++ import org.apache.commons.lang3.tuple.Pair; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; ++@@ -31,9 +30,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ import org.awaitility.Awaitility; ++ import org.junit.After; ++ import org.junit.Assert; ++-import org.junit.Assume; ++ import org.junit.Before; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++ public class TieredIndexFileTest { ++@@ -45,11 +42,12 @@ public class TieredIndexFileTest { ++ @Before ++ public void setUp() { ++ storeConfig = new TieredMessageStoreConfig(); +++ storeConfig.setBrokerName("IndexFileBroker"); ++ storeConfig.setStorePathRootDir(storePath); ++- storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment"); ++- storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2); ++- storeConfig.setTieredStoreIndexFileMaxIndexNum(3); ++- mq = new MessageQueue("TieredIndexFileTest", storeConfig.getBrokerName(), 1); +++ storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment"); +++ storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5); +++ storeConfig.setTieredStoreIndexFileMaxIndexNum(20); +++ mq = new MessageQueue("IndexFileTest", storeConfig.getBrokerName(), 1); ++ TieredStoreUtil.getMetadataStore(storeConfig); ++ TieredStoreExecutor.init(); ++ } ++@@ -61,77 +59,33 @@ public class TieredIndexFileTest { ++ TieredStoreExecutor.shutdown(); ++ } ++ ++- @Ignore ++ @Test ++ public void testAppendAndQuery() throws IOException, ClassNotFoundException, NoSuchMethodException { ++ if (Platform.isWindows()) { ++ return; ++ } ++ ++- // skip this test on windows ++- Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); ++- ++ TieredFileAllocator fileQueueFactory = new TieredFileAllocator(storeConfig); ++ TieredIndexFile indexFile = new TieredIndexFile(fileQueueFactory, storePath); +++ ++ indexFile.append(mq, 0, "key3", 3, 300, 1000); ++ indexFile.append(mq, 0, "key2", 2, 200, 1100); ++ indexFile.append(mq, 0, "key1", 1, 100, 1200); ++ ++- Awaitility.waitAtMost(5, TimeUnit.SECONDS) ++- .until(() -> { ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++- if (indexList.size() != 1) { ++- return false; ++- } ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 2, indexBuffer.remaining()); ++- ++- Assert.assertEquals(1, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++- ++- Assert.assertEquals(3, indexBuffer.getLong(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4)); ++- Assert.assertEquals(300, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8 + 4)); ++- return true; ++- }); ++- ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- ++- Awaitility.waitAtMost(5, TimeUnit.SECONDS) ++- .until(() -> { ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1300, 1300).join(); ++- if (indexList.size() != 1) { ++- return false; ++- } ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); ++- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++- return true; ++- }); ++- ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1300, 1300).join(); +++ // do not do schedule task here +++ TieredStoreExecutor.shutdown(); +++ List> indexList = +++ indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++ Assert.assertEquals(0, indexList.size()); ++ ++- indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1200, 1300).join(); ++- Assert.assertEquals(2, indexList.size()); ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); ++- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +++ // do compaction once +++ TieredStoreExecutor.init(); +++ storeConfig.setTieredStoreIndexFileRollingIdleInterval(0); +++ indexFile.doScheduleTask(); +++ Awaitility.await().atMost(Duration.ofSeconds(10)) +++ .until(() -> !indexFile.getPreMappedFile().getFile().exists()); ++ ++- indexBuffer = indexList.get(1).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE, indexBuffer.remaining()); ++- Assert.assertEquals(2, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); +++ Assert.assertEquals(1, indexList.size()); ++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 69c26d3d29cde7b4484ecd112ab9224f9f42bf45 Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Wed, 23 Aug 2023 10:27:52 +0800 ++Subject: [PATCH 5/8] [ISSUE #7228] Converge the use of some important ++ variables for some class ++ ++--- ++ .../apache/rocketmq/store/ConsumeQueue.java | 16 ++++++------ ++ .../rocketmq/store/MappedFileQueue.java | 26 +++++++++++-------- ++ .../store/MultiPathMappedFileQueue.java | 4 +-- ++ 3 files changed, 24 insertions(+), 22 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++index a0b886eb0..56bee2af3 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++@@ -145,7 +145,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ if (offset >= 0 && size > 0) { ++ mappedFileOffset = i + CQ_STORE_UNIT_SIZE; ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++ } ++@@ -409,7 +409,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ int logicFileSize = this.mappedFileSize; ++ ++- this.maxPhysicOffset = phyOffset; +++ this.setMaxPhysicOffset(phyOffset); ++ long maxExtAddr = 1; ++ boolean shouldDeleteFile = false; ++ while (true) { ++@@ -435,7 +435,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ mappedFile.setWrotePosition(pos); ++ mappedFile.setCommittedPosition(pos); ++ mappedFile.setFlushedPosition(pos); ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ // This maybe not take effect, when not every consume queue has extend file. ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++@@ -453,7 +453,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ mappedFile.setWrotePosition(pos); ++ mappedFile.setCommittedPosition(pos); ++ mappedFile.setFlushedPosition(pos); ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++ } ++@@ -881,8 +881,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, ++ final long cqOffset) { ++ ++- if (offset + size <= this.maxPhysicOffset) { ++- log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset); +++ if (offset + size <= this.getMaxPhysicOffset()) { +++ log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", this.getMaxPhysicOffset(), offset); ++ return true; ++ } ++ ++@@ -926,7 +926,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ); ++ } ++ } ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ return mappedFile.appendMessage(this.byteBufferIndex.array()); ++ } ++ return false; ++@@ -1130,7 +1130,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ @Override ++ public void destroy() { ++- this.maxPhysicOffset = -1; +++ this.setMaxPhysicOffset(-1); ++ this.minLogicOffset = 0; ++ this.mappedFileQueue.destroy(); ++ if (isExtReadEnable()) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++index 0bc70642f..32b90d14f 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++@@ -285,7 +285,7 @@ public class MappedFileQueue implements Swappable { ++ if (this.mappedFiles.isEmpty()) ++ return 0; ++ ++- long committed = this.flushedWhere; +++ long committed = this.getFlushedWhere(); ++ if (committed != 0) { ++ MappedFile mappedFile = this.getLastMappedFile(0, false); ++ if (mappedFile != null) { ++@@ -442,11 +442,11 @@ public class MappedFileQueue implements Swappable { ++ } ++ ++ public long remainHowManyDataToCommit() { ++- return getMaxWrotePosition() - committedWhere; +++ return getMaxWrotePosition() - getCommittedWhere(); ++ } ++ ++ public long remainHowManyDataToFlush() { ++- return getMaxOffset() - flushedWhere; +++ return getMaxOffset() - this.getFlushedWhere(); ++ } ++ ++ public void deleteLastMappedFile() { ++@@ -616,15 +616,15 @@ public class MappedFileQueue implements Swappable { ++ ++ public boolean flush(final int flushLeastPages) { ++ boolean result = true; ++- MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0); +++ MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0); ++ if (mappedFile != null) { ++ long tmpTimeStamp = mappedFile.getStoreTimestamp(); ++ int offset = mappedFile.flush(flushLeastPages); ++ long where = mappedFile.getFileFromOffset() + offset; ++- result = where == this.flushedWhere; ++- this.flushedWhere = where; +++ result = where == this.getFlushedWhere(); +++ this.setFlushedWhere(where); ++ if (0 == flushLeastPages) { ++- this.storeTimestamp = tmpTimeStamp; +++ this.setStoreTimestamp(tmpTimeStamp); ++ } ++ } ++ ++@@ -633,12 +633,12 @@ public class MappedFileQueue implements Swappable { ++ ++ public synchronized boolean commit(final int commitLeastPages) { ++ boolean result = true; ++- MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0); +++ MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0); ++ if (mappedFile != null) { ++ int offset = mappedFile.commit(commitLeastPages); ++ long where = mappedFile.getFileFromOffset() + offset; ++- result = where == this.committedWhere; ++- this.committedWhere = where; +++ result = where == this.getCommittedWhere(); +++ this.setCommittedWhere(where); ++ } ++ ++ return result; ++@@ -763,7 +763,7 @@ public class MappedFileQueue implements Swappable { ++ mf.destroy(1000 * 3); ++ } ++ this.mappedFiles.clear(); ++- this.flushedWhere = 0; +++ this.setFlushedWhere(0); ++ ++ // delete parent directory ++ File file = new File(storePath); ++@@ -848,6 +848,10 @@ public class MappedFileQueue implements Swappable { ++ return storeTimestamp; ++ } ++ +++ public void setStoreTimestamp(long storeTimestamp) { +++ this.storeTimestamp = storeTimestamp; +++ } +++ ++ public List getMappedFiles() { ++ return mappedFiles; ++ } ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++index 8f5af9438..8ff050dfe 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.store; ++ ++- ++ import java.util.Arrays; ++ import java.util.HashSet; ++ import java.util.Set; ++@@ -113,8 +112,7 @@ public class MultiPathMappedFileQueue extends MappedFileQueue { ++ mf.destroy(1000 * 3); ++ } ++ this.mappedFiles.clear(); ++- this.flushedWhere = 0; ++- +++ this.setFlushedWhere(0); ++ ++ Set storePathSet = getPaths(); ++ storePathSet.addAll(getReadonlyPaths()); ++-- ++2.32.0.windows.2 ++ ++ ++From 3884f595949462044c5cb3c236199bc1d7ad2341 Mon Sep 17 00:00:00 2001 ++From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= ++ ++Date: Wed, 23 Aug 2023 11:10:30 +0800 ++Subject: [PATCH 6/8] [ISSUE #7149] When creating and updating Topic, there ++ will be problems with permission settings (#7151) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings ++ ++* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings ++ ++* [issue#7249] ++ ++--------- ++ ++Co-authored-by: 十真 ++--- ++ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 3 ++- ++ 1 file changed, 2 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 13f9d002b..e8f943702 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1733,7 +1733,8 @@ public class BrokerController { ++ new TopicConfig(topicConfig.getTopicName(), ++ topicConfig.getReadQueueNums(), ++ topicConfig.getWriteQueueNums(), ++- this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); +++ topicConfig.getPerm() +++ & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); ++ } else { ++ registerTopicConfig = new TopicConfig(topicConfig); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 017ad110475e8024585327b44f47e5e97aabc63b Mon Sep 17 00:00:00 2001 ++From: echooymxq ++Date: Wed, 23 Aug 2023 11:11:42 +0800 ++Subject: [PATCH 7/8] [ISSUE #7219] Fix Concurrent modify syncStateSet and Mark ++ synchronizing frequently when shrink. (#7220) ++ ++--- ++ .../broker/controller/ReplicasManager.java | 29 ++++++++++--------- ++ .../ha/autoswitch/AutoSwitchHAService.java | 21 ++++++++------ ++ 2 files changed, 28 insertions(+), 22 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++index abae7cdb0..37c82e434 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++@@ -542,7 +542,7 @@ public class ReplicasManager { ++ this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId()); ++ this.tempBrokerMetadata.clear(); ++ this.brokerControllerId = this.brokerMetadata.getBrokerId(); ++- this.haService.setBrokerControllerId(this.brokerControllerId); +++ this.haService.setLocalBrokerId(this.brokerControllerId); ++ return true; ++ } catch (Exception e) { ++ LOGGER.error("fail to create metadata file", e); ++@@ -594,7 +594,7 @@ public class ReplicasManager { ++ if (this.brokerMetadata.isLoaded()) { ++ this.registerState = RegisterState.CREATE_METADATA_FILE_DONE; ++ this.brokerControllerId = brokerMetadata.getBrokerId(); ++- this.haService.setBrokerControllerId(this.brokerControllerId); +++ this.haService.setLocalBrokerId(this.brokerControllerId); ++ return; ++ } ++ // 2. check if temp metadata exist ++@@ -735,23 +735,26 @@ public class ReplicasManager { ++ if (this.checkSyncStateSetTaskFuture != null) { ++ this.checkSyncStateSetTaskFuture.cancel(false); ++ } ++- this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(() -> { ++- checkSyncStateSetAndDoReport(); ++- }, 3 * 1000, this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); +++ this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000, +++ this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); ++ } ++ ++ private void checkSyncStateSetAndDoReport() { ++- final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); ++- newSyncStateSet.add(this.brokerControllerId); ++- synchronized (this) { ++- if (this.syncStateSet != null) { ++- // Check if syncStateSet changed ++- if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { ++- return; +++ try { +++ final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); +++ newSyncStateSet.add(this.brokerControllerId); +++ synchronized (this) { +++ if (this.syncStateSet != null) { +++ // Check if syncStateSet changed +++ if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { +++ return; +++ } ++ } ++ } +++ doReportSyncStateSetChanged(newSyncStateSet); +++ } catch (Exception e) { +++ LOGGER.error("Check syncStateSet error", e); ++ } ++- doReportSyncStateSetChanged(newSyncStateSet); ++ } ++ ++ private void doReportSyncStateSetChanged(Set newSyncStateSet) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++index 6dc734e0c..d5393fdca 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++@@ -41,6 +41,7 @@ import java.nio.channels.SocketChannel; ++ import java.util.ArrayList; ++ import java.util.HashSet; ++ import java.util.List; +++import java.util.Iterator; ++ import java.util.Map; ++ import java.util.Objects; ++ import java.util.Set; ++@@ -73,7 +74,7 @@ public class AutoSwitchHAService extends DefaultHAService { ++ private EpochFileCache epochCache; ++ private AutoSwitchHAClient haClient; ++ ++- private Long brokerControllerId = null; +++ private Long localBrokerId = null; ++ ++ public AutoSwitchHAService() { ++ } ++@@ -287,9 +288,11 @@ public class AutoSwitchHAService extends DefaultHAService { ++ ++ // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable, ++ // it means that the broker has not connected. ++- for (Long slaveBrokerId : newSyncStateSet) { ++- if (!this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { ++- newSyncStateSet.remove(slaveBrokerId); +++ Iterator iterator = newSyncStateSet.iterator(); +++ while (iterator.hasNext()) { +++ Long slaveBrokerId = iterator.next(); +++ if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { +++ iterator.remove(); ++ isSyncStateSetChanged = true; ++ } ++ } ++@@ -419,7 +422,7 @@ public class AutoSwitchHAService extends DefaultHAService { ++ // To avoid the syncStateSet is not consistent with connectionList. ++ // Fix issue: https://github.com/apache/rocketmq/issues/6662 ++ for (Long syncId : currentSyncStateSet) { ++- if (!idList.contains(syncId) && this.brokerControllerId != null && !Objects.equals(syncId, this.brokerControllerId)) { +++ if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) { ++ LOGGER.warn("Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.", syncId); ++ // Without check and re-compute, return the confirmOffset's value directly. ++ return this.defaultMessageStore.getConfirmOffsetDirectly(); ++@@ -545,12 +548,12 @@ public class AutoSwitchHAService extends DefaultHAService { ++ return this.epochCache.getAllEntries(); ++ } ++ ++- public Long getBrokerControllerId() { ++- return brokerControllerId; +++ public Long getLocalBrokerId() { +++ return localBrokerId; ++ } ++ ++- public void setBrokerControllerId(Long brokerControllerId) { ++- this.brokerControllerId = brokerControllerId; +++ public void setLocalBrokerId(Long localBrokerId) { +++ this.localBrokerId = localBrokerId; ++ } ++ ++ class AutoSwitchAcceptSocketService extends AcceptSocketService { ++-- ++2.32.0.windows.2 ++ ++ ++From 77e8e54b37c3fc3ea0beffc1ace6f5bf20af10d9 Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Wed, 23 Aug 2023 15:56:39 +0800 ++Subject: [PATCH 8/8] [ISSUE #7223] Support batch ack for grpc client in proxy ++ (#7225) ++ ++--- ++ .../client/impl/mqclient/MQClientAPIExt.java | 26 +++ ++ .../rocketmq/proxy/config/ProxyConfig.java | 10 + ++ .../grpc/v2/consumer/AckMessageActivity.java | 136 ++++++++--- ++ .../proxy/processor/AbstractProcessor.java | 4 +- ++ .../proxy/processor/BatchAckResult.java | 53 +++++ ++ .../proxy/processor/ConsumerProcessor.java | 64 +++++ ++ .../processor/DefaultMessagingProcessor.java | 7 + ++ .../proxy/processor/MessagingProcessor.java | 18 ++ ++ .../message/ClusterMessageService.java | 16 +- ++ .../service/message/LocalMessageService.java | 58 +++++ ++ .../proxy/service/message/MessageService.java | 8 + ++ .../service/message/ReceiptHandleMessage.java | 39 ++++ ++ .../v2/consumer/AckMessageActivityTest.java | 221 +++++++++++++++--- ++ .../proxy/processor/BaseProcessorTest.java | 18 +- ++ .../processor/ConsumerProcessorTest.java | 115 +++++++++ ++ .../service/mqclient/MQClientAPIExtTest.java | 12 + ++ 16 files changed, 728 insertions(+), 77 deletions(-) ++ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++index fb8f8d11f..d7c8ef8d9 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++@@ -306,6 +306,32 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ return future; ++ } ++ +++ public CompletableFuture batchAckMessageAsync( +++ String brokerAddr, +++ String topic, +++ String consumerGroup, +++ List extraInfoList, +++ long timeoutMillis +++ ) { +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() { +++ @Override +++ public void onSuccess(AckResult ackResult) { +++ future.complete(ackResult); +++ } +++ +++ @Override +++ public void onException(Throwable t) { +++ future.completeExceptionally(t); +++ } +++ }, topic, consumerGroup, extraInfoList); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ ++ public CompletableFuture changeInvisibleTimeAsync( ++ String brokerAddr, ++ String brokerName, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 39caaa0d9..76a243919 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -250,6 +250,8 @@ public class ProxyConfig implements ConfigFile { ++ private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000; ++ private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000; ++ +++ private boolean enableBatchAck = false; +++ ++ @Override ++ public void initData() { ++ parseDelayLevel(); ++@@ -1379,4 +1381,12 @@ public class ProxyConfig implements ConfigFile { ++ public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) { ++ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; ++ } +++ +++ public boolean isEnableBatchAck() { +++ return enableBatchAck; +++ } +++ +++ public void setEnableBatchAck(boolean enableBatchAck) { +++ this.enableBatchAck = enableBatchAck; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++index 9a3a77201..97c716c8f 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++@@ -31,12 +31,15 @@ import org.apache.rocketmq.client.consumer.AckStatus; ++ import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; +++import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; ++ import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; ++ import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +++import org.apache.rocketmq.proxy.processor.BatchAckResult; ++ import org.apache.rocketmq.proxy.processor.MessagingProcessor; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ ++ public class AckMessageActivity extends AbstractMessingActivity { ++ ++@@ -50,60 +53,98 @@ public class AckMessageActivity extends AbstractMessingActivity { ++ ++ try { ++ validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); ++- ++- CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; ++- for (int i = 0; i < request.getEntriesCount(); i++) { ++- futures[i] = processAckMessage(ctx, request, request.getEntries(i)); +++ String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); +++ String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()); +++ if (ConfigurationManager.getProxyConfig().isEnableBatchAck()) { +++ future = ackMessageInBatch(ctx, group, topic, request); +++ } else { +++ future = ackMessageOneByOne(ctx, group, topic, request); ++ } ++- CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { ++- if (throwable != null) { ++- future.completeExceptionally(throwable); ++- return; ++- } +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ +++ protected CompletableFuture ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) { +++ List handleMessageList = new ArrayList<>(request.getEntriesCount()); ++ +++ for (AckMessageEntry ackMessageEntry : request.getEntriesList()) { +++ String handleString = getHandleString(ctx, group, request, ackMessageEntry); +++ handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId())); +++ } +++ return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic) +++ .thenApply(batchAckResultList -> { +++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder(); ++ Set responseCodes = new HashSet<>(); ++- List entryList = new ArrayList<>(); ++- for (CompletableFuture entryFuture : futures) { ++- AckMessageResultEntry entryResult = entryFuture.join(); ++- responseCodes.add(entryResult.getStatus().getCode()); ++- entryList.add(entryResult); ++- } ++- AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() ++- .addAllEntries(entryList); ++- if (responseCodes.size() > 1) { ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); ++- } else if (responseCodes.size() == 1) { ++- Code code = responseCodes.stream().findAny().get(); ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); ++- } else { ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult); +++ responseBuilder.addEntries(entry); +++ responseCodes.add(entry.getStatus().getCode()); ++ } ++- future.complete(responseBuilder.build()); +++ setAckResponseStatus(responseBuilder, responseCodes); +++ return responseBuilder.build(); ++ }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); +++ } +++ +++ protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) { +++ ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage(); +++ AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder() +++ .setMessageId(handleMessage.getMessageId()) +++ .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle()); +++ if (batchAckResult.getProxyException() != null) { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException())); +++ } else { +++ AckResult ackResult = batchAckResult.getAckResult(); +++ if (AckStatus.OK.equals(ackResult.getStatus())) { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); +++ } else { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")); +++ } ++ } ++- return future; +++ return resultBuilder.build(); ++ } ++ ++- protected CompletableFuture processAckMessage(ProxyContext ctx, AckMessageRequest request, +++ protected CompletableFuture ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) { +++ CompletableFuture resultFuture = new CompletableFuture<>(); +++ CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; +++ for (int i = 0; i < request.getEntriesCount(); i++) { +++ futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i)); +++ } +++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +++ if (throwable != null) { +++ resultFuture.completeExceptionally(throwable); +++ return; +++ } +++ +++ Set responseCodes = new HashSet<>(); +++ List entryList = new ArrayList<>(); +++ for (CompletableFuture entryFuture : futures) { +++ AckMessageResultEntry entryResult = entryFuture.join(); +++ responseCodes.add(entryResult.getStatus().getCode()); +++ entryList.add(entryResult); +++ } +++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() +++ .addAllEntries(entryList); +++ setAckResponseStatus(responseBuilder, responseCodes); +++ resultFuture.complete(responseBuilder.build()); +++ }); +++ return resultFuture; +++ } +++ +++ protected CompletableFuture processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request, ++ AckMessageEntry ackMessageEntry) { ++ CompletableFuture future = new CompletableFuture<>(); ++ ++ try { ++- String handleString = ackMessageEntry.getReceiptHandle(); ++- ++- String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); ++- MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); ++- if (messageReceiptHandle != null) { ++- handleString = messageReceiptHandle.getReceiptHandleStr(); ++- } +++ String handleString = this.getHandleString(ctx, group, request, ackMessageEntry); ++ CompletableFuture ackResultFuture = this.messagingProcessor.ackMessage( ++ ctx, ++ ReceiptHandle.decode(handleString), ++ ackMessageEntry.getMessageId(), ++ group, ++- GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); +++ topic +++ ); ++ ackResultFuture.thenAccept(result -> { ++ future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result)); ++ }).exceptionally(t -> { ++@@ -139,4 +180,25 @@ public class AckMessageActivity extends AbstractMessingActivity { ++ .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")) ++ .build(); ++ } +++ +++ protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set responseCodes) { +++ if (responseCodes.size() > 1) { +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); +++ } else if (responseCodes.size() == 1) { +++ Code code = responseCodes.stream().findAny().get(); +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); +++ } else { +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); +++ } +++ } +++ +++ protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) { +++ String handleString = ackMessageEntry.getReceiptHandle(); +++ +++ MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); +++ if (messageReceiptHandle != null) { +++ handleString = messageReceiptHandle.getReceiptHandleStr(); +++ } +++ return handleString; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++index b61c3df9e..c63212c23 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++@@ -27,6 +27,8 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { ++ protected MessagingProcessor messagingProcessor; ++ protected ServiceManager serviceManager; ++ +++ protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); +++ ++ public AbstractProcessor(MessagingProcessor messagingProcessor, ++ ServiceManager serviceManager) { ++ this.messagingProcessor = messagingProcessor; ++@@ -35,7 +37,7 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { ++ ++ protected void validateReceiptHandle(ReceiptHandle handle) { ++ if (handle.isExpired()) { ++- throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); +++ throw EXPIRED_HANDLE_PROXY_EXCEPTION; ++ } ++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++new file mode 100644 ++index 000000000..dfb9c9b9e ++--- /dev/null +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++@@ -0,0 +1,53 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.proxy.processor; +++ +++import org.apache.rocketmq.client.consumer.AckResult; +++import org.apache.rocketmq.proxy.common.ProxyException; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; +++ +++public class BatchAckResult { +++ +++ private final ReceiptHandleMessage receiptHandleMessage; +++ private AckResult ackResult; +++ private ProxyException proxyException; +++ +++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, +++ AckResult ackResult) { +++ this.receiptHandleMessage = receiptHandleMessage; +++ this.ackResult = ackResult; +++ } +++ +++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, +++ ProxyException proxyException) { +++ this.receiptHandleMessage = receiptHandleMessage; +++ this.proxyException = proxyException; +++ } +++ +++ public ReceiptHandleMessage getReceiptHandleMessage() { +++ return receiptHandleMessage; +++ } +++ +++ public AckResult getAckResult() { +++ return ackResult; +++ } +++ +++ public ProxyException getProxyException() { +++ return proxyException; +++ } +++} ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++index 656a6339d..f3522b374 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++@@ -48,6 +48,7 @@ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.common.utils.ProxyUtils; ++ import org.apache.rocketmq.proxy.service.ServiceManager; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; ++@@ -241,6 +242,69 @@ public class ConsumerProcessor extends AbstractProcessor { ++ return FutureUtils.addExecutor(future, this.executor); ++ } ++ +++ public CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ) { +++ CompletableFuture> future = new CompletableFuture<>(); +++ try { +++ List batchAckResultList = new ArrayList<>(handleMessageList.size()); +++ Map> brokerHandleListMap = new HashMap<>(); +++ +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ if (handleMessage.getReceiptHandle().isExpired()) { +++ batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION)); +++ continue; +++ } +++ List brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>()); +++ brokerHandleList.add(handleMessage); +++ } +++ +++ if (brokerHandleListMap.isEmpty()) { +++ return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor); +++ } +++ Set>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet(); +++ CompletableFuture>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()]; +++ int futureIndex = 0; +++ for (Map.Entry> entry : brokerHandleListMapEntrySet) { +++ futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis); +++ } +++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +++ if (throwable != null) { +++ future.completeExceptionally(throwable); +++ } +++ for (CompletableFuture> resultFuture : futures) { +++ batchAckResultList.addAll(resultFuture.join()); +++ } +++ future.complete(batchAckResultList); +++ }); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return FutureUtils.addExecutor(future, this.executor); +++ } +++ +++ protected CompletableFuture> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List handleMessageList, long timeoutMillis) { +++ return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis) +++ .thenApply(result -> { +++ List results = new ArrayList<>(); +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ results.add(new BatchAckResult(handleMessage, result)); +++ } +++ return results; +++ }) +++ .exceptionally(throwable -> { +++ List results = new ArrayList<>(); +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable))); +++ } +++ return results; +++ }); +++ } +++ ++ public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, ++ String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++index 188cb7b9b..ba150051b 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++@@ -46,6 +46,7 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.service.ServiceManager; ++ import org.apache.rocketmq.proxy.service.ServiceManagerFactory; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; ++@@ -183,6 +184,12 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen ++ return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis); ++ } ++ +++ @Override +++ public CompletableFuture> batchAckMessage(ProxyContext ctx, +++ List handleMessageList, String consumerGroup, String topic, long timeoutMillis) { +++ return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis); +++ } +++ ++ @Override ++ public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, ++ String groupName, String topicName, long invisibleTime, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++index d86be0bd8..2ae7418ba 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++@@ -37,6 +37,7 @@ import org.apache.rocketmq.proxy.common.Address; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; ++@@ -155,6 +156,23 @@ public interface MessagingProcessor extends StartAndShutdown { ++ long timeoutMillis ++ ); ++ +++ default CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic +++ ) { +++ return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS); +++ } +++ +++ CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ); +++ ++ default CompletableFuture changeInvisibleTime( ++ ProxyContext ctx, ++ ReceiptHandle handle, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++index 9f163f1b9..70b72deae 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++@@ -20,9 +20,11 @@ import com.google.common.collect.Lists; ++ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; +++import java.util.stream.Collectors; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.PopResult; ++ import org.apache.rocketmq.client.consumer.PullResult; +++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ import org.apache.rocketmq.common.message.Message; ++@@ -31,7 +33,6 @@ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.TopicRouteService; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -137,6 +138,19 @@ public class ClusterMessageService implements MessageService { ++ ); ++ } ++ +++ @Override +++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, String consumerGroup, +++ String topic, long timeoutMillis) { +++ List extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList()); +++ return this.mqClientAPIFactory.getClient().batchAckMessageAsync( +++ this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()), +++ topic, +++ consumerGroup, +++ extraInfoList, +++ timeoutMillis +++ ); +++ } +++ ++ @Override ++ public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, ++ PullMessageRequestHeader requestHeader, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++index eb2c4d9ee..ca7dcc9eb 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.service.message; ++ import io.netty.channel.ChannelHandlerContext; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; +++import java.util.BitSet; ++ import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.List; ++@@ -54,6 +55,8 @@ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; +++import org.apache.rocketmq.remoting.protocol.body.BatchAck; +++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; ++@@ -364,6 +367,61 @@ public class LocalMessageService implements MessageService { ++ }); ++ } ++ +++ @Override +++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, +++ String consumerGroup, String topic, long timeoutMillis) { +++ SimpleChannel channel = channelManager.createChannel(ctx); +++ ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); +++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); +++ +++ Map batchAckMap = new HashMap<>(); +++ for (ReceiptHandleMessage receiptHandleMessage : handleList) { +++ String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle(); +++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo); +++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + +++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" + +++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + +++ ExtraInfoUtil.getPopTime(extraInfoData); +++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { +++ BatchAck newBatchAck = new BatchAck(); +++ newBatchAck.setConsumerGroup(consumerGroup); +++ newBatchAck.setTopic(topic); +++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); +++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); +++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); +++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); +++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); +++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); +++ newBatchAck.setBitSet(new BitSet()); +++ return newBatchAck; +++ }); +++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); +++ } +++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); +++ requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName()); +++ requestBody.setAcks(new ArrayList<>(batchAckMap.values())); +++ +++ command.setBody(requestBody.encode()); +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ RemotingCommand response = brokerController.getAckMessageProcessor() +++ .processRequest(channelHandlerContext, command); +++ future.complete(response); +++ } catch (Exception e) { +++ log.error("Fail to process batchAckMessage command", e); +++ future.completeExceptionally(e); +++ } +++ return future.thenApply(r -> { +++ AckResult ackResult = new AckResult(); +++ if (ResponseCode.SUCCESS == r.getCode()) { +++ ackResult.setStatus(AckStatus.OK); +++ } else { +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ } +++ return ackResult; +++ }); +++ } +++ ++ @Override ++ public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, ++ PullMessageRequestHeader requestHeader, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++index 15da17154..58a835adb 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++@@ -91,6 +91,14 @@ public interface MessageService { ++ long timeoutMillis ++ ); ++ +++ CompletableFuture batchAckMessage( +++ ProxyContext ctx, +++ List handleList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ); +++ ++ CompletableFuture pullMessage( ++ ProxyContext ctx, ++ AddressableMessageQueue messageQueue, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++new file mode 100644 ++index 000000000..ae63fed49 ++--- /dev/null +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++@@ -0,0 +1,39 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.proxy.service.message; +++ +++import org.apache.rocketmq.common.consumer.ReceiptHandle; +++ +++public class ReceiptHandleMessage { +++ +++ private final ReceiptHandle receiptHandle; +++ private final String messageId; +++ +++ public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) { +++ this.receiptHandle = receiptHandle; +++ this.messageId = messageId; +++ } +++ +++ public ReceiptHandle getReceiptHandle() { +++ return receiptHandle; +++ } +++ +++ public String getMessageId() { +++ return messageId; +++ } +++} ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++index 49fdfc6a8..3c4746105 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++@@ -20,21 +20,32 @@ package org.apache.rocketmq.proxy.grpc.v2.consumer; ++ import apache.rocketmq.v2.AckMessageEntry; ++ import apache.rocketmq.v2.AckMessageRequest; ++ import apache.rocketmq.v2.AckMessageResponse; +++import apache.rocketmq.v2.AckMessageResultEntry; ++ import apache.rocketmq.v2.Code; ++ import apache.rocketmq.v2.Resource; +++import java.util.ArrayList; +++import java.util.HashMap; +++import java.util.List; +++import java.util.Map; ++ import java.util.concurrent.CompletableFuture; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.AckStatus; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +++import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +++import org.apache.rocketmq.proxy.processor.BatchAckResult; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.junit.Before; ++ import org.junit.Test; +++import org.mockito.stubbing.Answer; ++ ++ import static org.junit.Assert.assertEquals; ++ import static org.mockito.ArgumentMatchers.any; +++import static org.mockito.ArgumentMatchers.anyList; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; +++import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.when; ++ ++ public class AckMessageActivityTest extends BaseActivityTest { ++@@ -52,43 +63,197 @@ public class AckMessageActivityTest extends BaseActivityTest { ++ ++ @Test ++ public void testAckMessage() throws Throwable { ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString())) +++ ConfigurationManager.getProxyConfig().setEnableBatchAck(false); +++ +++ String msg1 = "msg1"; +++ String msg2 = "msg2"; +++ String msg3 = "msg3"; +++ +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString())) ++ .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired")); ++ ++ AckResult msg2AckResult = new AckResult(); ++ msg2AckResult.setStatus(AckStatus.OK); ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString())) +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString())) ++ .thenReturn(CompletableFuture.completedFuture(msg2AckResult)); ++ ++ AckResult msg3AckResult = new AckResult(); ++ msg3AckResult.setStatus(AckStatus.NO_EXIST); ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString())) +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString())) ++ .thenReturn(CompletableFuture.completedFuture(msg3AckResult)); ++ ++- AckMessageResponse response = this.ackMessageActivity.ackMessage( ++- createContext(), ++- AckMessageRequest.newBuilder() ++- .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++- .setGroup(Resource.newBuilder().setName(GROUP).build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg1") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++- .build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg2") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++- .build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg3") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++- .build()) ++- .build() ++- ).get(); ++- ++- assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); ++- assertEquals(3, response.getEntriesCount()); ++- assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); ++- assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); ++- assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg1) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg2) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.OK, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg3) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg1) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg2) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg3) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ +++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +++ assertEquals(3, response.getEntriesCount()); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); +++ assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); +++ } +++ } +++ +++ @Test +++ public void testAckMessageInBatch() throws Throwable { +++ ConfigurationManager.getProxyConfig().setEnableBatchAck(true); +++ +++ String successMessageId = "msg1"; +++ String notOkMessageId = "msg2"; +++ String exceptionMessageId = "msg3"; +++ +++ doAnswer((Answer>>) invocation -> { +++ List receiptHandleMessageList = invocation.getArgument(1, List.class); +++ List batchAckResultList = new ArrayList<>(); +++ for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) { +++ BatchAckResult batchAckResult; +++ if (receiptHandleMessage.getMessageId().equals(successMessageId)) { +++ AckResult ackResult = new AckResult(); +++ ackResult.setStatus(AckStatus.OK); +++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); +++ } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) { +++ AckResult ackResult = new AckResult(); +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); +++ } else { +++ batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "")); +++ } +++ batchAckResultList.add(batchAckResult); +++ } +++ return CompletableFuture.completedFuture(batchAckResultList); +++ }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString()); +++ +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(successMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.OK, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(notOkMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(exceptionMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(successMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(notOkMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(exceptionMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ +++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +++ assertEquals(3, response.getEntriesCount()); +++ Map msgCode = new HashMap<>(); +++ for (AckMessageResultEntry entry : response.getEntriesList()) { +++ msgCode.put(entry.getMessageId(), entry.getStatus().getCode()); +++ } +++ assertEquals(Code.OK, msgCode.get(successMessageId)); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId)); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId)); +++ } ++ } ++ } ++\ No newline at end of file ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++index 5c1ea9627..072630e39 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++@@ -66,14 +66,6 @@ public class BaseProcessorTest extends InitConfigTest { ++ protected ProxyRelayService proxyRelayService; ++ @Mock ++ protected MetadataService metadataService; ++- @Mock ++- protected ProducerProcessor producerProcessor; ++- @Mock ++- protected ConsumerProcessor consumerProcessor; ++- @Mock ++- protected TransactionProcessor transactionProcessor; ++- @Mock ++- protected ClientProcessor clientProcessor; ++ ++ public void before() throws Throwable { ++ super.before(); ++@@ -92,6 +84,13 @@ public class BaseProcessorTest extends InitConfigTest { ++ } ++ ++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) { +++ return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(), +++ RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), +++ RANDOM.nextInt(Integer.MAX_VALUE), "mockBroker"); +++ } +++ +++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime, +++ long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) { ++ MessageExt messageExt = new MessageExt(); ++ messageExt.setTopic(topic); ++ messageExt.setTags(tags); ++@@ -100,8 +99,7 @@ public class BaseProcessorTest extends InitConfigTest { ++ messageExt.setMsgId(MessageClientIDSetter.createUniqID()); ++ messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE)); ++ MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, ++- ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime, ++- RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE))); +++ ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset)); ++ return messageExt; ++ } ++ ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++index 717e86fc0..db268a06e 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++@@ -20,8 +20,11 @@ package org.apache.rocketmq.proxy.processor; ++ import com.google.common.collect.Sets; ++ import java.time.Duration; ++ import java.util.ArrayList; +++import java.util.Collections; +++import java.util.HashMap; ++ import java.util.HashSet; ++ import java.util.List; +++import java.util.Map; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.Executors; ++@@ -39,7 +42,10 @@ import org.apache.rocketmq.common.message.MessageClientIDSetter; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.proxy.common.ProxyContext; +++import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +++import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.common.utils.ProxyUtils; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.MessageQueueView; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -50,16 +56,22 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; ++ import org.junit.Before; ++ import org.junit.Test; ++ import org.mockito.ArgumentCaptor; +++import org.mockito.stubbing.Answer; ++ ++ import static org.assertj.core.api.Assertions.assertThat; ++ import static org.junit.Assert.assertEquals; ++ import static org.junit.Assert.assertNotNull; +++import static org.junit.Assert.assertNull; ++ import static org.junit.Assert.assertSame; ++ import static org.mockito.ArgumentMatchers.any; +++import static org.mockito.ArgumentMatchers.anyList; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; +++import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.mock; +++import static org.mockito.Mockito.never; +++import static org.mockito.Mockito.verify; ++ import static org.mockito.Mockito.when; ++ ++ public class ConsumerProcessorTest extends BaseProcessorTest { ++@@ -162,6 +174,109 @@ public class ConsumerProcessorTest extends BaseProcessorTest { ++ assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo()); ++ } ++ +++ @Test +++ public void testBatchAckExpireMessage() throws Throwable { +++ String brokerName1 = "brokerName1"; +++ +++ List receiptHandleMessageList = new ArrayList<>(); +++ for (int i = 0; i < 3; i++) { +++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, +++ 0, 0, 0, i, brokerName1); +++ ReceiptHandle expireHandle = create(expireMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); +++ } +++ +++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); +++ +++ verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); +++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ assertNull(batchAckResult.getAckResult()); +++ assertNotNull(batchAckResult.getProxyException()); +++ assertNotNull(batchAckResult.getReceiptHandleMessage()); +++ } +++ +++ } +++ +++ @Test +++ public void testBatchAckMessage() throws Throwable { +++ String brokerName1 = "brokerName1"; +++ String brokerName2 = "brokerName2"; +++ String errThrowBrokerName = "errThrowBrokerName"; +++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, +++ 0, 0, 0, 0, brokerName1); +++ ReceiptHandle expireHandle = create(expireMessage); +++ +++ List receiptHandleMessageList = new ArrayList<>(); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); +++ List broker1Msg = new ArrayList<>(); +++ List broker2Msg = new ArrayList<>(); +++ +++ long now = System.currentTimeMillis(); +++ int msgNum = 3; +++ for (int i = 0; i < msgNum; i++) { +++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, i + 1, brokerName1); +++ ReceiptHandle brokerHandle = create(brokerMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); +++ broker1Msg.add(brokerMessage.getMsgId()); +++ } +++ for (int i = 0; i < msgNum; i++) { +++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, i + 1, brokerName2); +++ ReceiptHandle brokerHandle = create(brokerMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); +++ broker2Msg.add(brokerMessage.getMsgId()); +++ } +++ +++ // for this message, will throw exception in batchAckMessage +++ MessageExt errThrowMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, 0, errThrowBrokerName); +++ ReceiptHandle errThrowHandle = create(errThrowMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId())); +++ +++ Collections.shuffle(receiptHandleMessageList); +++ +++ doAnswer((Answer>) invocation -> { +++ List handleMessageList = invocation.getArgument(1, List.class); +++ AckResult ackResult = new AckResult(); +++ String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName(); +++ if (brokerName.equals(brokerName1)) { +++ ackResult.setStatus(AckStatus.OK); +++ } else if (brokerName.equals(brokerName2)) { +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ } else { +++ return FutureUtils.completeExceptionally(new RuntimeException()); +++ } +++ +++ return CompletableFuture.completedFuture(ackResult); +++ }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); +++ +++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); +++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); +++ +++ // check ackResult for each msg +++ Map msgBatchAckResult = new HashMap<>(); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult); +++ } +++ for (String msgId : broker1Msg) { +++ assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus()); +++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); +++ } +++ for (String msgId : broker2Msg) { +++ assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus()); +++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); +++ } +++ assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException()); +++ assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode()); +++ assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult()); +++ +++ assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException()); +++ assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode()); +++ assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult()); +++ } +++ ++ @Test ++ public void testChangeInvisibleTime() throws Throwable { ++ ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000)); ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++index 77a119a29..3f3a4ae40 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++@@ -220,6 +220,18 @@ public class MQClientAPIExtTest { ++ assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get()); ++ } ++ +++ @Test +++ public void testBatchAckMessageAsync() throws Exception { +++ AckResult ackResult = new AckResult(); +++ doAnswer((Answer) mock -> { +++ AckCallback ackCallback = mock.getArgument(2); +++ ackCallback.onSuccess(ackResult); +++ return null; +++ }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any()); +++ +++ assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get()); +++ } +++ ++ @Test ++ public void testChangeInvisibleTimeAsync() throws Exception { ++ AckResult ackResult = new AckResult(); ++-- ++2.32.0.windows.2 ++ +diff --git a/patch013-backport-enhance-admin-output.patch b/patch013-backport-enhance-admin-output.patch +new file mode 100644 +index 000000000..3fa60916f +--- /dev/null ++++ b/patch013-backport-enhance-admin-output.patch +@@ -0,0 +1,892 @@ ++From 7e018520ef707a841c66c55d621f6560d03b631b Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Fri, 25 Aug 2023 09:49:22 +0800 ++Subject: [PATCH 1/6] Add expireAfterAccess for cache (#7247) ++ ++Add expireAfterAccess for cache ++--- ++ .../rocketmq/proxy/config/ProxyConfig.java | 59 ++++++++++++++----- ++ .../metadata/ClusterMetadataService.java | 6 +- ++ .../service/route/TopicRouteService.java | 14 +++-- ++ 3 files changed, 56 insertions(+), 23 deletions(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 76a243919..2994893d7 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -155,14 +155,17 @@ public class ProxyConfig implements ConfigFile { ++ private int consumerProcessorThreadPoolQueueCapacity = 10000; ++ ++ private boolean useEndpointPortFromRequest = false; ++- private int topicRouteServiceCacheExpiredInSeconds = 20; +++ +++ private int topicRouteServiceCacheExpiredSeconds = 300; +++ private int topicRouteServiceCacheRefreshSeconds = 20; ++ private int topicRouteServiceCacheMaxNum = 20000; ++ private int topicRouteServiceThreadPoolNums = PROCESSOR_NUMBER; ++ private int topicRouteServiceThreadPoolQueueCapacity = 5000; ++- ++- private int topicConfigCacheExpiredInSeconds = 20; +++ private int topicConfigCacheExpiredSeconds = 300; +++ private int topicConfigCacheRefreshSeconds = 20; ++ private int topicConfigCacheMaxNum = 20000; ++- private int subscriptionGroupConfigCacheExpiredInSeconds = 20; +++ private int subscriptionGroupConfigCacheExpiredSeconds = 300; +++ private int subscriptionGroupConfigCacheRefreshSeconds = 20; ++ private int subscriptionGroupConfigCacheMaxNum = 20000; ++ private int metadataThreadPoolNums = 3; ++ private int metadataThreadPoolQueueCapacity = 100000; ++@@ -794,12 +797,20 @@ public class ProxyConfig implements ConfigFile { ++ this.consumerProcessorThreadPoolQueueCapacity = consumerProcessorThreadPoolQueueCapacity; ++ } ++ ++- public int getTopicRouteServiceCacheExpiredInSeconds() { ++- return topicRouteServiceCacheExpiredInSeconds; +++ public int getTopicRouteServiceCacheExpiredSeconds() { +++ return topicRouteServiceCacheExpiredSeconds; +++ } +++ +++ public void setTopicRouteServiceCacheExpiredSeconds(int topicRouteServiceCacheExpiredSeconds) { +++ this.topicRouteServiceCacheExpiredSeconds = topicRouteServiceCacheExpiredSeconds; ++ } ++ ++- public void setTopicRouteServiceCacheExpiredInSeconds(int topicRouteServiceCacheExpiredInSeconds) { ++- this.topicRouteServiceCacheExpiredInSeconds = topicRouteServiceCacheExpiredInSeconds; +++ public int getTopicRouteServiceCacheRefreshSeconds() { +++ return topicRouteServiceCacheRefreshSeconds; +++ } +++ +++ public void setTopicRouteServiceCacheRefreshSeconds(int topicRouteServiceCacheRefreshSeconds) { +++ this.topicRouteServiceCacheRefreshSeconds = topicRouteServiceCacheRefreshSeconds; ++ } ++ ++ public int getTopicRouteServiceCacheMaxNum() { ++@@ -826,12 +837,20 @@ public class ProxyConfig implements ConfigFile { ++ this.topicRouteServiceThreadPoolQueueCapacity = topicRouteServiceThreadPoolQueueCapacity; ++ } ++ ++- public int getTopicConfigCacheExpiredInSeconds() { ++- return topicConfigCacheExpiredInSeconds; +++ public int getTopicConfigCacheRefreshSeconds() { +++ return topicConfigCacheRefreshSeconds; +++ } +++ +++ public void setTopicConfigCacheRefreshSeconds(int topicConfigCacheRefreshSeconds) { +++ this.topicConfigCacheRefreshSeconds = topicConfigCacheRefreshSeconds; +++ } +++ +++ public int getTopicConfigCacheExpiredSeconds() { +++ return topicConfigCacheExpiredSeconds; ++ } ++ ++- public void setTopicConfigCacheExpiredInSeconds(int topicConfigCacheExpiredInSeconds) { ++- this.topicConfigCacheExpiredInSeconds = topicConfigCacheExpiredInSeconds; +++ public void setTopicConfigCacheExpiredSeconds(int topicConfigCacheExpiredSeconds) { +++ this.topicConfigCacheExpiredSeconds = topicConfigCacheExpiredSeconds; ++ } ++ ++ public int getTopicConfigCacheMaxNum() { ++@@ -842,12 +861,20 @@ public class ProxyConfig implements ConfigFile { ++ this.topicConfigCacheMaxNum = topicConfigCacheMaxNum; ++ } ++ ++- public int getSubscriptionGroupConfigCacheExpiredInSeconds() { ++- return subscriptionGroupConfigCacheExpiredInSeconds; +++ public int getSubscriptionGroupConfigCacheRefreshSeconds() { +++ return subscriptionGroupConfigCacheRefreshSeconds; +++ } +++ +++ public void setSubscriptionGroupConfigCacheRefreshSeconds(int subscriptionGroupConfigCacheRefreshSeconds) { +++ this.subscriptionGroupConfigCacheRefreshSeconds = subscriptionGroupConfigCacheRefreshSeconds; +++ } +++ +++ public int getSubscriptionGroupConfigCacheExpiredSeconds() { +++ return subscriptionGroupConfigCacheExpiredSeconds; ++ } ++ ++- public void setSubscriptionGroupConfigCacheExpiredInSeconds(int subscriptionGroupConfigCacheExpiredInSeconds) { ++- this.subscriptionGroupConfigCacheExpiredInSeconds = subscriptionGroupConfigCacheExpiredInSeconds; +++ public void setSubscriptionGroupConfigCacheExpiredSeconds(int subscriptionGroupConfigCacheExpiredSeconds) { +++ this.subscriptionGroupConfigCacheExpiredSeconds = subscriptionGroupConfigCacheExpiredSeconds; ++ } ++ ++ public int getSubscriptionGroupConfigCacheMaxNum() { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java ++index bc9582ad8..d34a0efd9 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java ++@@ -69,11 +69,13 @@ public class ClusterMetadataService extends AbstractStartAndShutdown implements ++ ); ++ this.topicConfigCache = CacheBuilder.newBuilder() ++ .maximumSize(config.getTopicConfigCacheMaxNum()) ++- .refreshAfterWrite(config.getTopicConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) +++ .expireAfterAccess(config.getTopicConfigCacheExpiredSeconds(), TimeUnit.SECONDS) +++ .refreshAfterWrite(config.getTopicConfigCacheRefreshSeconds(), TimeUnit.SECONDS) ++ .build(new ClusterTopicConfigCacheLoader()); ++ this.subscriptionGroupConfigCache = CacheBuilder.newBuilder() ++ .maximumSize(config.getSubscriptionGroupConfigCacheMaxNum()) ++- .refreshAfterWrite(config.getSubscriptionGroupConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) +++ .expireAfterAccess(config.getSubscriptionGroupConfigCacheExpiredSeconds(), TimeUnit.SECONDS) +++ .refreshAfterWrite(config.getSubscriptionGroupConfigCacheRefreshSeconds(), TimeUnit.SECONDS) ++ .build(new ClusterSubscriptionGroupConfigCacheLoader()); ++ ++ this.init(); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++index e012a5465..84348adc3 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++@@ -68,10 +68,13 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ ); ++ this.mqClientAPIFactory = mqClientAPIFactory; ++ ++- this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum()). ++- refreshAfterWrite(config.getTopicRouteServiceCacheExpiredInSeconds(), TimeUnit.SECONDS). ++- executor(cacheRefreshExecutor).build(new CacheLoader() { ++- @Override public @Nullable MessageQueueView load(String topic) throws Exception { +++ this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum()) +++ .expireAfterAccess(config.getTopicRouteServiceCacheExpiredSeconds(), TimeUnit.SECONDS) +++ .refreshAfterWrite(config.getTopicRouteServiceCacheRefreshSeconds(), TimeUnit.SECONDS) +++ .executor(cacheRefreshExecutor) +++ .build(new CacheLoader() { +++ @Override +++ public @Nullable MessageQueueView load(String topic) throws Exception { ++ try { ++ TopicRouteData topicRouteData = mqClientAPIFactory.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis()); ++ return buildMessageQueueView(topic, topicRouteData); ++@@ -83,7 +86,8 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ } ++ } ++ ++- @Override public @Nullable MessageQueueView reload(@NonNull String key, +++ @Override +++ public @Nullable MessageQueueView reload(@NonNull String key, ++ @NonNull MessageQueueView oldValue) throws Exception { ++ try { ++ return load(key); ++-- ++2.32.0.windows.2 ++ ++ ++From 5f6dc90f9dab35809fcb0407d4d5cc2737d2335e Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Fri, 25 Aug 2023 11:17:23 +0800 ++Subject: [PATCH 2/6] [ISSUE #7250] Beautify command rocksDBConfigToJson output ++ ++Co-authored-by: Ziy1-Tan ++--- ++ .../metadata/RocksDBConfigToJsonCommand.java | 32 +++++++++++-------- ++ 1 file changed, 18 insertions(+), 14 deletions(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++index 3053f4684..3fc63e4dd 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++@@ -21,13 +21,13 @@ import org.apache.commons.cli.CommandLine; ++ import org.apache.commons.cli.Option; ++ import org.apache.commons.cli.Options; ++ import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.common.UtilAll; ++ import org.apache.rocketmq.common.config.RocksDBConfigManager; ++ import org.apache.rocketmq.common.utils.DataConverter; ++ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.tools.command.SubCommand; ++ import org.apache.rocketmq.tools.command.SubCommandException; ++ ++-import java.io.File; ++ import java.util.HashMap; ++ import java.util.Map; ++ ++@@ -48,7 +48,7 @@ public class RocksDBConfigToJsonCommand implements SubCommand { ++ @Override ++ public Options buildCommandlineOptions(Options options) { ++ Option pathOption = new Option("p", "path", true, ++- "Absolute path to the metadata directory"); +++ "Absolute path for the metadata directory"); ++ pathOption.setRequired(true); ++ options.addOption(pathOption); ++ ++@@ -63,15 +63,14 @@ public class RocksDBConfigToJsonCommand implements SubCommand { ++ @Override ++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { ++ String path = commandLine.getOptionValue("path").trim(); ++- if (StringUtils.isEmpty(path) || !new File(path).exists()) { +++ if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { ++ System.out.print("Rocksdb path is invalid.\n"); ++ return; ++ } ++ ++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); ++ ++- final long memTableFlushInterval = 60 * 60 * 1000L; ++- RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); +++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(60 * 60 * 1000L); ++ try { ++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for topics.json ++@@ -84,13 +83,16 @@ public class RocksDBConfigToJsonCommand implements SubCommand { ++ topicConfigTable.put(topic, jsonObject); ++ }); ++ ++- if (isLoad) { ++- topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); ++- final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); ++- System.out.print(topicsJsonStr + "\n"); +++ if (!isLoad) { +++ System.out.print("RocksDB load error, path=" + path); ++ return; ++ } +++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); +++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); +++ System.out.print(topicsJsonStr + "\n"); +++ return; ++ } +++ ++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for subscriptionGroup.json ++ final Map subscriptionGroupJsonConfig = new HashMap<>(); ++@@ -102,13 +104,15 @@ public class RocksDBConfigToJsonCommand implements SubCommand { ++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); ++ }); ++ ++- if (isLoad) { ++- subscriptionGroupJsonConfig.put("subscriptionGroupTable", ++- (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); ++- final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); ++- System.out.print(subscriptionGroupJsonStr + "\n"); +++ if (!isLoad) { +++ System.out.print("RocksDB load error, path=" + path); ++ return; ++ } +++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", +++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); +++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); +++ System.out.print(subscriptionGroupJsonStr + "\n"); +++ return; ++ } ++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); ++ } finally { ++-- ++2.32.0.windows.2 ++ ++ ++From b4f73e2aabc1b141cec98431899e4090340adf0f Mon Sep 17 00:00:00 2001 ++From: mxsm ++Date: Sun, 27 Aug 2023 20:58:58 +0800 ++Subject: [PATCH 3/6] [ISSUE #7271] Optimize the configuration for setting the ++ quantity of TimerDequeuePutMessageService (#7272) ++ ++--- ++ .../java/org/apache/rocketmq/store/timer/TimerMessageStore.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++index 690f4863e..181f7087a 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++@@ -222,7 +222,7 @@ public class TimerMessageStore { ++ dequeueGetMessageServices[i] = new TimerDequeueGetMessageService(); ++ } ++ ++- int putThreadNum = Math.max(storeConfig.getTimerGetMessageThreadNum(), 1); +++ int putThreadNum = Math.max(storeConfig.getTimerPutMessageThreadNum(), 1); ++ dequeuePutMessageServices = new TimerDequeuePutMessageService[putThreadNum]; ++ for (int i = 0; i < dequeuePutMessageServices.length; i++) { ++ dequeuePutMessageServices[i] = new TimerDequeuePutMessageService(); ++-- ++2.32.0.windows.2 ++ ++ ++From 3e100103af68588528bf32f3752a85e8023f46f8 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Tue, 29 Aug 2023 13:48:51 +0800 ++Subject: [PATCH 4/6] [ISSUE #7277] Enhance rocksDBConfigToJson to support ++ metadata counting (#7276) ++ ++--- ++ .../common/config/AbstractRocksDBStorage.java | 4 +- ++ .../common/config/ConfigRocksDBStorage.java | 6 + ++ .../tools/command/MQAdminStartup.java | 4 +- ++ .../ExportMetadataInRocksDBCommand.java | 138 ++++++++++++++++++ ++ .../metadata/RocksDBConfigToJsonCommand.java | 122 ---------------- ++ ...> ExportMetadataInRocksDBCommandTest.java} | 38 +++-- ++ 6 files changed, 173 insertions(+), 139 deletions(-) ++ create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java ++ delete mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++ rename tools/src/test/java/org/apache/rocketmq/tools/command/metadata/{KvConfigToJsonCommandTest.java => ExportMetadataInRocksDBCommandTest.java} (62%) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++index e3673baad..a720a5be3 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++@@ -385,8 +385,10 @@ public abstract class AbstractRocksDBStorage { ++ this.options.close(); ++ } ++ //4. close db. ++- if (db != null) { +++ if (db != null && !this.readOnly) { ++ this.db.syncWal(); +++ } +++ if (db != null) { ++ this.db.closeE(); ++ } ++ //5. help gc. ++diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java ++index 9d05ed282..463bd8fed 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +++++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java ++@@ -60,6 +60,12 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage { ++ this.readOnly = false; ++ } ++ +++ public ConfigRocksDBStorage(final String dbPath, boolean readOnly) { +++ super(); +++ this.dbPath = dbPath; +++ this.readOnly = readOnly; +++ } +++ ++ private void initOptions() { ++ this.options = createConfigDBOptions(); ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++index 324aa1856..788fa83c2 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++@@ -80,7 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; ++ import org.apache.rocketmq.tools.command.message.SendMessageCommand; ++-import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; +++import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand; ++ import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; ++ import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; ++ import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; ++@@ -212,7 +212,6 @@ public class MQAdminStartup { ++ ++ initCommand(new ClusterListSubCommand()); ++ initCommand(new TopicListSubCommand()); ++- initCommand(new RocksDBConfigToJsonCommand()); ++ ++ initCommand(new UpdateKvConfigCommand()); ++ initCommand(new DeleteKvConfigCommand()); ++@@ -257,6 +256,7 @@ public class MQAdminStartup { ++ initCommand(new ExportMetadataCommand()); ++ initCommand(new ExportConfigsCommand()); ++ initCommand(new ExportMetricsCommand()); +++ initCommand(new ExportMetadataInRocksDBCommand()); ++ ++ initCommand(new HAStatusSubCommand()); ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java ++new file mode 100644 ++index 000000000..2a7d3fba4 ++--- /dev/null +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java ++@@ -0,0 +1,138 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.export; +++ +++import com.alibaba.fastjson.JSONObject; +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.Option; +++import org.apache.commons.cli.Options; +++import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.common.UtilAll; +++import org.apache.rocketmq.common.config.ConfigRocksDBStorage; +++import org.apache.rocketmq.common.utils.DataConverter; +++import org.apache.rocketmq.remoting.RPCHook; +++import org.apache.rocketmq.tools.command.SubCommand; +++import org.apache.rocketmq.tools.command.SubCommandException; +++import org.rocksdb.RocksIterator; +++ +++import java.util.HashMap; +++import java.util.Map; +++import java.util.concurrent.atomic.AtomicLong; +++import java.util.function.BiConsumer; +++ +++public class ExportMetadataInRocksDBCommand implements SubCommand { +++ private static final String TOPICS_JSON_CONFIG = "topics"; +++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; +++ +++ @Override +++ public String commandName() { +++ return "exportMetadataInRocksDB"; +++ } +++ +++ @Override +++ public String commandDesc() { +++ return "export RocksDB kv config (topics/subscriptionGroups)"; +++ } +++ +++ @Override +++ public Options buildCommandlineOptions(Options options) { +++ Option pathOption = new Option("p", "path", true, +++ "Absolute path for the metadata directory"); +++ pathOption.setRequired(true); +++ options.addOption(pathOption); +++ +++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + +++ "topics/subscriptionGroups"); +++ configTypeOption.setRequired(true); +++ options.addOption(configTypeOption); +++ +++ Option jsonEnableOption = new Option("j", "jsonEnable", true, +++ "Json format enable, Default: false"); +++ jsonEnableOption.setRequired(false); +++ options.addOption(jsonEnableOption); +++ +++ return options; +++ } +++ +++ @Override +++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { +++ String path = commandLine.getOptionValue("path").trim(); +++ if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { +++ System.out.print("RocksDB path is invalid.\n"); +++ return; +++ } +++ +++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); +++ +++ boolean jsonEnable = false; +++ if (commandLine.hasOption("jsonEnable")) { +++ jsonEnable = Boolean.parseBoolean(commandLine.getOptionValue("jsonEnable").trim()); +++ } +++ +++ +++ ConfigRocksDBStorage kvStore = new ConfigRocksDBStorage(path, true /* readOnly */); +++ if (!kvStore.start()) { +++ System.out.print("RocksDB load error, path=" + path + "\n"); +++ return; +++ } +++ +++ try { +++ if (TOPICS_JSON_CONFIG.equalsIgnoreCase(configType) || SUBSCRIPTION_GROUP_JSON_CONFIG.equalsIgnoreCase(configType)) { +++ handleExportMetadata(kvStore, configType, jsonEnable); +++ } else { +++ System.out.printf("Invalid config type=%s, Options: topics,subscriptionGroups\n", configType); +++ } +++ } finally { +++ kvStore.shutdown(); +++ } +++ } +++ +++ private static void handleExportMetadata(ConfigRocksDBStorage kvStore, String configType, boolean jsonEnable) { +++ if (jsonEnable) { +++ final Map jsonConfig = new HashMap<>(); +++ final Map configTable = new HashMap<>(); +++ iterateKvStore(kvStore, (key, value) -> { +++ final String configKey = new String(key, DataConverter.charset); +++ final String configValue = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(configValue); +++ configTable.put(configKey, jsonObject); +++ } +++ ); +++ +++ jsonConfig.put(configType.equalsIgnoreCase(TOPICS_JSON_CONFIG) ? "topicConfigTable" : "subscriptionGroupTable", +++ (JSONObject) JSONObject.toJSON(configTable)); +++ final String jsonConfigStr = JSONObject.toJSONString(jsonConfig, true); +++ System.out.print(jsonConfigStr + "\n"); +++ } else { +++ AtomicLong count = new AtomicLong(0); +++ iterateKvStore(kvStore, (key, value) -> { +++ final String configKey = new String(key, DataConverter.charset); +++ final String configValue = new String(value, DataConverter.charset); +++ System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), configKey, configValue); +++ }); +++ } +++ } +++ +++ private static void iterateKvStore(ConfigRocksDBStorage kvStore, BiConsumer biConsumer) { +++ try (RocksIterator iterator = kvStore.iterator()) { +++ iterator.seekToFirst(); +++ for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { +++ biConsumer.accept(iterator.key(), iterator.value()); +++ } +++ } +++ } +++} ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++deleted file mode 100644 ++index 3fc63e4dd..000000000 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +++++ /dev/null ++@@ -1,122 +0,0 @@ ++-/* ++- * Licensed to the Apache Software Foundation (ASF) under one or more ++- * contributor license agreements. See the NOTICE file distributed with ++- * this work for additional information regarding copyright ownership. ++- * The ASF licenses this file to You under the Apache License, Version 2.0 ++- * (the "License"); you may not use this file except in compliance with ++- * the License. You may obtain a copy of the License at ++- * ++- * http://www.apache.org/licenses/LICENSE-2.0 ++- * ++- * Unless required by applicable law or agreed to in writing, software ++- * distributed under the License is distributed on an "AS IS" BASIS, ++- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++- * See the License for the specific language governing permissions and ++- * limitations under the License. ++- */ ++-package org.apache.rocketmq.tools.command.metadata; ++- ++-import com.alibaba.fastjson.JSONObject; ++-import org.apache.commons.cli.CommandLine; ++-import org.apache.commons.cli.Option; ++-import org.apache.commons.cli.Options; ++-import org.apache.commons.lang3.StringUtils; ++-import org.apache.rocketmq.common.UtilAll; ++-import org.apache.rocketmq.common.config.RocksDBConfigManager; ++-import org.apache.rocketmq.common.utils.DataConverter; ++-import org.apache.rocketmq.remoting.RPCHook; ++-import org.apache.rocketmq.tools.command.SubCommand; ++-import org.apache.rocketmq.tools.command.SubCommandException; ++- ++-import java.util.HashMap; ++-import java.util.Map; ++- ++-public class RocksDBConfigToJsonCommand implements SubCommand { ++- private static final String TOPICS_JSON_CONFIG = "topics"; ++- private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; ++- ++- @Override ++- public String commandName() { ++- return "rocksDBConfigToJson"; ++- } ++- ++- @Override ++- public String commandDesc() { ++- return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; ++- } ++- ++- @Override ++- public Options buildCommandlineOptions(Options options) { ++- Option pathOption = new Option("p", "path", true, ++- "Absolute path for the metadata directory"); ++- pathOption.setRequired(true); ++- options.addOption(pathOption); ++- ++- Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + ++- "topics/subscriptionGroups"); ++- configTypeOption.setRequired(true); ++- options.addOption(configTypeOption); ++- ++- return options; ++- } ++- ++- @Override ++- public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { ++- String path = commandLine.getOptionValue("path").trim(); ++- if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { ++- System.out.print("Rocksdb path is invalid.\n"); ++- return; ++- } ++- ++- String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); ++- ++- RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(60 * 60 * 1000L); ++- try { ++- if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { ++- // for topics.json ++- final Map topicsJsonConfig = new HashMap<>(); ++- final Map topicConfigTable = new HashMap<>(); ++- boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++- final String topic = new String(key, DataConverter.charset); ++- final String topicConfig = new String(value, DataConverter.charset); ++- final JSONObject jsonObject = JSONObject.parseObject(topicConfig); ++- topicConfigTable.put(topic, jsonObject); ++- }); ++- ++- if (!isLoad) { ++- System.out.print("RocksDB load error, path=" + path); ++- return; ++- } ++- topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); ++- final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); ++- System.out.print(topicsJsonStr + "\n"); ++- return; ++- } ++- ++- if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { ++- // for subscriptionGroup.json ++- final Map subscriptionGroupJsonConfig = new HashMap<>(); ++- final Map subscriptionGroupTable = new HashMap<>(); ++- boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++- final String subscriptionGroup = new String(key, DataConverter.charset); ++- final String subscriptionGroupConfig = new String(value, DataConverter.charset); ++- final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); ++- subscriptionGroupTable.put(subscriptionGroup, jsonObject); ++- }); ++- ++- if (!isLoad) { ++- System.out.print("RocksDB load error, path=" + path); ++- return; ++- } ++- subscriptionGroupJsonConfig.put("subscriptionGroupTable", ++- (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); ++- final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); ++- System.out.print(subscriptionGroupJsonStr + "\n"); ++- return; ++- } ++- System.out.print("Config type was not recognized, configType=" + configType + "\n"); ++- } finally { ++- kvConfigManager.stop(); ++- } ++- } ++-} ++diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java ++similarity index 62% ++rename from tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++rename to tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java ++index b2f66c7b0..2b938c90f 100644 ++--- a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java +++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java ++@@ -21,43 +21,53 @@ import org.apache.commons.cli.DefaultParser; ++ import org.apache.commons.cli.Options; ++ import org.apache.rocketmq.srvutil.ServerUtil; ++ import org.apache.rocketmq.tools.command.SubCommandException; +++import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand; ++ import org.junit.Test; ++ ++ import java.io.File; ++ ++ import static org.assertj.core.api.Assertions.assertThat; ++ ++-public class KvConfigToJsonCommandTest { +++public class ExportMetadataInRocksDBCommandTest { ++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; ++ ++ @Test ++ public void testExecute() throws SubCommandException { ++ { ++- String[] cases = new String[]{"topics", "subscriptionGroups"}; ++- for (String c : cases) { ++- RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ String[][] cases = new String[][] { +++ {"topics", "false"}, +++ {"topics", "false1"}, +++ {"topics", "true"}, +++ {"subscriptionGroups", "false"}, +++ {"subscriptionGroups", "false2"}, +++ {"subscriptionGroups", "true"} +++ }; +++ +++ for (String[] c : cases) { +++ ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand(); ++ Options options = ServerUtil.buildCommandlineOptions(new Options()); ++- String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c}; +++ String[] subargs = new String[] {"-p " + BASE_PATH + c[0], "-t " + c[0], "-j " + c[1]}; ++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, ++- cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.execute(commandLine, options, null); ++- assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c); ++- assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c); +++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c[0]); +++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c[0]); +++ assertThat(commandLine.getOptionValue("j").trim()).isEqualTo(c[1]); ++ } ++ } ++ // invalid cases ++ { ++- String[][] cases = new String[][]{ ++- {"-p " + BASE_PATH + "tmpPath", "-t topics"}, ++- {"-p ", "-t topics"}, ++- {"-p " + BASE_PATH + "topics", "-t invalid_type"} +++ String[][] cases = new String[][] { +++ {"-p " + BASE_PATH + "tmpPath", "-t topics", "-j true"}, +++ {"-p ", "-t topics", "-j true"}, +++ {"-p " + BASE_PATH + "topics", "-t invalid_type", "-j true"} ++ }; ++ ++ for (String[] c : cases) { ++- RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand(); ++ Options options = ServerUtil.buildCommandlineOptions(new Options()); ++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, ++- cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.execute(commandLine, options, null); ++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From fa549154370cb866a90e37c13a90d2c598d6b1f6 Mon Sep 17 00:00:00 2001 ++From: yuz10 <845238369@qq.com> ++Date: Tue, 29 Aug 2023 15:22:09 +0800 ++Subject: [PATCH 5/6] [ISSUE #7261] Slave high CPU usage when ++ enableScheduleAsyncDeliver=true (#7262) ++ ++* [ISSUE #6390] Add break to the exception of WHEEL_TIMER_NOT_ENABLE. ++ ++* fix broker start fail if mapped file size is 0 ++ ++* log ++ ++* only delete the last empty file ++ ++* change dataReadAheadEnable default to true ++ ++* fix endless loop when master change to slave. ++--- ++ .../rocketmq/broker/schedule/ScheduleMessageService.java | 7 ++++++- ++ 1 file changed, 6 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++index aed0ee19f..297b14207 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++@@ -566,7 +566,8 @@ public class ScheduleMessageService extends ConfigManager { ++ pendingQueue.remove(); ++ break; ++ case RUNNING: ++- break; +++ scheduleNextTask(); +++ return; ++ case EXCEPTION: ++ if (!isStarted()) { ++ log.warn("HandlePutResultTask shutdown, info={}", putResultProcess.toString()); ++@@ -586,6 +587,10 @@ public class ScheduleMessageService extends ConfigManager { ++ } ++ } ++ +++ scheduleNextTask(); +++ } +++ +++ private void scheduleNextTask() { ++ if (isStarted()) { ++ ScheduleMessageService.this.handleExecutorService ++ .schedule(new HandlePutResultTask(this.delayLevel), DELAY_FOR_A_SLEEP, TimeUnit.MILLISECONDS); ++-- ++2.32.0.windows.2 ++ ++ ++From 9f34f55e1dac495730c9cd5469f2ab3225b8f0b9 Mon Sep 17 00:00:00 2001 ++From: ShuangxiDing ++Date: Tue, 29 Aug 2023 15:48:46 +0800 ++Subject: [PATCH 6/6] [ISSUE #7226] Filter tlvs in ppv2 which contents not are ++ spec-compliant ASCII characters and space (#7227) ++ ++Filter tlvs in ppv2 which not are spec-compliant ASCII characters and space ++--- ++ .../rocketmq/common/utils/BinaryUtil.java | 17 +++++++++++++++++ ++ .../grpc/ProxyAndTlsProtocolNegotiator.java | 8 +++++++- ++ .../remoting/netty/NettyRemotingServer.java | 8 +++++++- ++ 3 files changed, 31 insertions(+), 2 deletions(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java ++index 421adaca4..7b4b24819 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java +++++ b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java ++@@ -43,4 +43,21 @@ public class BinaryUtil { ++ byte[] bytes = calculateMd5(content); ++ return Hex.encodeHexString(bytes, false); ++ } +++ +++ /** +++ * Returns true if subject contains only bytes that are spec-compliant ASCII characters. +++ * @param subject +++ * @return +++ */ +++ public static boolean isAscii(byte[] subject) { +++ if (subject == null) { +++ return false; +++ } +++ for (byte b : subject) { +++ if ((b & 0x80) != 0) { +++ return false; +++ } +++ } +++ return true; +++ } ++ } ++\ No newline at end of file ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java ++index ee167bd7b..b584ddfbd 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java ++@@ -24,6 +24,7 @@ import io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiator; ++ import io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiators; ++ import io.grpc.netty.shaded.io.grpc.netty.ProtocolNegotiationEvent; ++ import io.grpc.netty.shaded.io.netty.buffer.ByteBuf; +++import io.grpc.netty.shaded.io.netty.buffer.ByteBufUtil; ++ import io.grpc.netty.shaded.io.netty.channel.ChannelHandler; ++ import io.grpc.netty.shaded.io.netty.channel.ChannelHandlerContext; ++ import io.grpc.netty.shaded.io.netty.channel.ChannelInboundHandlerAdapter; ++@@ -44,6 +45,7 @@ import org.apache.commons.collections.CollectionUtils; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.constant.HAProxyConstants; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.BinaryUtil; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++@@ -191,9 +193,13 @@ public class ProxyAndTlsProtocolNegotiator implements InternalProtocolNegotiator ++ } ++ if (CollectionUtils.isNotEmpty(msg.tlvs())) { ++ msg.tlvs().forEach(tlv -> { +++ byte[] valueBytes = ByteBufUtil.getBytes(tlv.content()); +++ if (!BinaryUtil.isAscii(valueBytes)) { +++ return; +++ } ++ Attributes.Key key = AttributeKeys.valueOf( ++ HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); ++- String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8)); +++ String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8)); ++ builder.set(key, value); ++ }); ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++index 17f138f86..e626260c9 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++@@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty; ++ ++ import io.netty.bootstrap.ServerBootstrap; ++ import io.netty.buffer.ByteBuf; +++import io.netty.buffer.ByteBufUtil; ++ import io.netty.buffer.PooledByteBufAllocator; ++ import io.netty.channel.Channel; ++ import io.netty.channel.ChannelDuplexHandler; ++@@ -58,6 +59,7 @@ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.HAProxyConstants; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.BinaryUtil; ++ import org.apache.rocketmq.common.utils.NetworkUtil; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++@@ -787,9 +789,13 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ } ++ if (CollectionUtils.isNotEmpty(msg.tlvs())) { ++ msg.tlvs().forEach(tlv -> { +++ byte[] valueBytes = ByteBufUtil.getBytes(tlv.content()); +++ if (!BinaryUtil.isAscii(valueBytes)) { +++ return; +++ } ++ AttributeKey key = AttributeKeys.valueOf( ++ HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); ++- String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8)); +++ String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8)); ++ channel.attr(key).set(value); ++ }); ++ } ++-- ++2.32.0.windows.2 ++ +diff --git a/patch014-backport-Queue-Selection-Strategy-Optimization.patch b/patch014-backport-Queue-Selection-Strategy-Optimization.patch +new file mode 100644 +index 000000000..90d3553aa +--- /dev/null ++++ b/patch014-backport-Queue-Selection-Strategy-Optimization.patch +@@ -0,0 +1,2023 @@ ++From b028277018946868838a82a08211071bc231a175 Mon Sep 17 00:00:00 2001 ++From: Ji Juntao ++Date: Tue, 29 Aug 2023 16:13:38 +0800 ++Subject: [PATCH] [ISSUE #6567] [RIP-63] Queue Selection Strategy Optimization ++ (#6568) ++ ++Optimize the proxy's and client's selection strategy for brokers when sending messages, and use multiple selection strategies as a pipeline to filter suitable queues. ++--- ++ .../apache/rocketmq/client/ClientConfig.java | 54 +++++ ++ .../client/common/ThreadLocalIndex.java | 8 + ++ .../rocketmq/client/impl/MQClientAPIImpl.java | 12 +- ++ .../client/impl/factory/MQClientInstance.java | 7 + ++ .../impl/producer/DefaultMQProducerImpl.java | 87 ++++++-- ++ .../impl/producer/TopicPublishInfo.java | 40 ++++ ++ .../client/latency/LatencyFaultTolerance.java | 66 +++++- ++ .../latency/LatencyFaultToleranceImpl.java | 189 ++++++++++++++---- ++ .../client/latency/MQFaultStrategy.java | 155 ++++++++++---- ++ .../rocketmq/client/latency/Resolver.java | 17 +- ++ .../client/latency/ServiceDetector.java | 30 +++ ++ .../LatencyFaultToleranceImplTest.java | 36 +++- ++ .../processor/DefaultRequestProcessor.java | 24 --- ++ .../rocketmq/proxy/config/ProxyConfig.java | 46 +++++ ++ .../grpc/v2/producer/SendMessageActivity.java | 2 +- ++ .../proxy/processor/ProducerProcessor.java | 18 +- ++ .../service/route/LocalTopicRouteService.java | 2 +- ++ .../service/route/MessageQueueSelector.java | 95 ++++++++- ++ .../proxy/service/route/MessageQueueView.java | 18 +- ++ .../service/route/TopicRouteService.java | 80 +++++++- ++ .../consumer/ReceiveMessageActivityTest.java | 5 +- ++ .../v2/producer/SendMessageActivityTest.java | 82 +++++++- ++ .../proxy/service/BaseServiceTest.java | 4 +- ++ .../route/MessageQueueSelectorTest.java | 8 +- ++ .../sysmessage/HeartbeatSyncerTest.java | 2 +- ++ .../ClusterTransactionServiceTest.java | 8 +- ++ 26 files changed, 919 insertions(+), 176 deletions(-) ++ rename remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java => client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java (65%) ++ create mode 100644 client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++index f87450f66..bb0fe3522 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++@@ -38,6 +38,8 @@ public class ClientConfig { ++ public static final String SOCKS_PROXY_CONFIG = "com.rocketmq.socks.proxy.config"; ++ public static final String DECODE_READ_BODY = "com.rocketmq.read.body"; ++ public static final String DECODE_DECOMPRESS_BODY = "com.rocketmq.decompress.body"; +++ public static final String SEND_LATENCY_ENABLE = "com.rocketmq.sendLatencyEnable"; +++ public static final String START_DETECTOR_ENABLE = "com.rocketmq.startDetectorEnable"; ++ public static final String HEART_BEAT_V2 = "com.rocketmq.heartbeat.v2"; ++ private String namesrvAddr = NameServerAddressUtils.getNameServerAddresses(); ++ private String clientIP = NetworkUtil.getLocalAddress(); ++@@ -72,6 +74,8 @@ public class ClientConfig { ++ private String socksProxyConfig = System.getProperty(SOCKS_PROXY_CONFIG, "{}"); ++ ++ private int mqClientApiTimeout = 3 * 1000; +++ private int detectTimeout = 200; +++ private int detectInterval = 2 * 1000; ++ ++ private LanguageCode language = LanguageCode.JAVA; ++ ++@@ -81,6 +85,15 @@ public class ClientConfig { ++ */ ++ protected boolean enableStreamRequestType = false; ++ +++ /** +++ * Enable the fault tolerance mechanism of the client sending process. +++ * DO NOT OPEN when ORDER messages are required. +++ * Turning on will interfere with the queue selection functionality, +++ * possibly conflicting with the order message. +++ */ +++ private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false")); +++ private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false")); +++ ++ public String buildMQClientId() { ++ StringBuilder sb = new StringBuilder(); ++ sb.append(this.getClientIP()); ++@@ -186,6 +199,10 @@ public class ClientConfig { ++ this.decodeDecompressBody = cc.decodeDecompressBody; ++ this.enableStreamRequestType = cc.enableStreamRequestType; ++ this.useHeartbeatV2 = cc.useHeartbeatV2; +++ this.startDetectorEnable = cc.startDetectorEnable; +++ this.sendLatencyEnable = cc.sendLatencyEnable; +++ this.detectInterval = cc.detectInterval; +++ this.detectTimeout = cc.detectTimeout; ++ } ++ ++ public ClientConfig cloneClientConfig() { ++@@ -210,6 +227,10 @@ public class ClientConfig { ++ cc.decodeDecompressBody = decodeDecompressBody; ++ cc.enableStreamRequestType = enableStreamRequestType; ++ cc.useHeartbeatV2 = useHeartbeatV2; +++ cc.startDetectorEnable = startDetectorEnable; +++ cc.sendLatencyEnable = sendLatencyEnable; +++ cc.detectInterval = detectInterval; +++ cc.detectTimeout = detectTimeout; ++ return cc; ++ } ++ ++@@ -381,6 +402,38 @@ public class ClientConfig { ++ this.enableStreamRequestType = enableStreamRequestType; ++ } ++ +++ public boolean isSendLatencyEnable() { +++ return sendLatencyEnable; +++ } +++ +++ public void setSendLatencyEnable(boolean sendLatencyEnable) { +++ this.sendLatencyEnable = sendLatencyEnable; +++ } +++ +++ public boolean isStartDetectorEnable() { +++ return startDetectorEnable; +++ } +++ +++ public void setStartDetectorEnable(boolean startDetectorEnable) { +++ this.startDetectorEnable = startDetectorEnable; +++ } +++ +++ public int getDetectTimeout() { +++ return this.detectTimeout; +++ } +++ +++ public void setDetectTimeout(int detectTimeout) { +++ this.detectTimeout = detectTimeout; +++ } +++ +++ public int getDetectInterval() { +++ return this.detectInterval; +++ } +++ +++ public void setDetectInterval(int detectInterval) { +++ this.detectInterval = detectInterval; +++ } +++ ++ public boolean isUseHeartbeatV2() { ++ return useHeartbeatV2; ++ } ++@@ -403,6 +456,7 @@ public class ClientConfig { ++ + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name() ++ + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout ++ + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody +++ + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable ++ + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]"; ++ } ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java ++index 4a3d90135..3a086c13d 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java +++++ b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java ++@@ -33,6 +33,14 @@ public class ThreadLocalIndex { ++ return index & POSITIVE_MASK; ++ } ++ +++ public void reset() { +++ int index = Math.abs(random.nextInt(Integer.MAX_VALUE)); +++ if (index < 0) { +++ index = 0; +++ } +++ this.threadLocalIndex.set(index); +++ } +++ ++ @Override ++ public String toString() { ++ return "ThreadLocalIndex{" + ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 213c26fd6..3201a493f 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -666,7 +666,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ } catch (Throwable e) { ++ } ++ ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); ++ return; ++ } ++ ++@@ -684,14 +684,14 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ } catch (Throwable e) { ++ } ++ ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); ++ } catch (Exception e) { ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++ retryTimesWhenSendFailed, times, e, context, false, producer); ++ } ++ } else { ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); ++ if (!responseFuture.isSendRequestOK()) { ++ MQClientException ex = new MQClientException("send request failed", responseFuture.getCause()); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++@@ -711,7 +711,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ }); ++ } catch (Exception ex) { ++ long cost = System.currentTimeMillis() - beginStartTime; ++- producer.updateFaultItem(brokerName, cost, true); +++ producer.updateFaultItem(brokerName, cost, true, false); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++ retryTimesWhenSendFailed, times, ex, context, true, producer); ++ } ++@@ -735,7 +735,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ if (needRetry && tmp <= timesTotal) { ++ String retryBrokerName = brokerName;//by default, it will send to the same broker ++ if (topicPublishInfo != null) { //select one message queue accordingly, in order to determine which broker to send ++- MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName); +++ MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName, false); ++ retryBrokerName = instance.getBrokerNameFromMessageQueue(mqChosen); ++ } ++ String addr = instance.findBrokerAddressInPublish(retryBrokerName); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++index 8851bc815..9484b26f8 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++@@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ThreadFactory; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicLong; ++ import java.util.concurrent.locks.Lock; ++@@ -125,6 +126,12 @@ public class MQClientInstance { ++ private final Set brokerSupportV2HeartbeatSet = new HashSet(); ++ private final ConcurrentMap brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap(); ++ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "MQClientFactoryScheduledThread")); +++ private final ScheduledExecutorService fetchRemoteConfigExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { +++ @Override +++ public Thread newThread(Runnable r) { +++ return new Thread(r, "MQClientFactoryFetchRemoteConfigScheduledThread"); +++ } +++ }); ++ private final PullMessageService pullMessageService; ++ private final RebalanceService rebalanceService; ++ private final DefaultMQProducer defaultMQProducer; ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++index 3f4c6e5f7..bbbb17b07 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++@@ -33,6 +33,8 @@ import java.util.concurrent.RejectedExecutionException; ++ import java.util.concurrent.Semaphore; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; +++ +++import com.google.common.base.Optional; ++ import org.apache.rocketmq.client.QueryResult; ++ import org.apache.rocketmq.client.Validators; ++ import org.apache.rocketmq.client.common.ClientErrorCode; ++@@ -49,6 +51,8 @@ import org.apache.rocketmq.client.impl.CommunicationMode; ++ import org.apache.rocketmq.client.impl.MQClientManager; ++ import org.apache.rocketmq.client.impl.factory.MQClientInstance; ++ import org.apache.rocketmq.client.latency.MQFaultStrategy; +++import org.apache.rocketmq.client.latency.Resolver; +++import org.apache.rocketmq.client.latency.ServiceDetector; ++ import org.apache.rocketmq.client.producer.DefaultMQProducer; ++ import org.apache.rocketmq.client.producer.LocalTransactionExecuter; ++ import org.apache.rocketmq.client.producer.LocalTransactionState; ++@@ -112,7 +116,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ private ServiceState serviceState = ServiceState.CREATE_JUST; ++ private MQClientInstance mQClientFactory; ++ private ArrayList checkForbiddenHookList = new ArrayList<>(); ++- private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); +++ private MQFaultStrategy mqFaultStrategy; ++ private ExecutorService asyncSenderExecutor; ++ ++ // compression related ++@@ -153,8 +157,38 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ semaphoreAsyncSendSize = new Semaphore(1024 * 1024, true); ++ log.info("semaphoreAsyncSendSize can not be smaller than 1M."); ++ } ++- } ++ +++ ServiceDetector serviceDetector = new ServiceDetector() { +++ @Override +++ public boolean detect(String endpoint, long timeoutMillis) { +++ Optional candidateTopic = pickTopic(); +++ if (!candidateTopic.isPresent()) { +++ return false; +++ } +++ try { +++ MessageQueue mq = new MessageQueue(candidateTopic.get(), null, 0); +++ mQClientFactory.getMQClientAPIImpl() +++ .getMaxOffset(endpoint, mq, timeoutMillis); +++ return true; +++ } catch (Exception e) { +++ return false; +++ } +++ } +++ }; +++ +++ this.mqFaultStrategy = new MQFaultStrategy(defaultMQProducer.cloneClientConfig(), new Resolver() { +++ @Override +++ public String resolve(String name) { +++ return DefaultMQProducerImpl.this.mQClientFactory.findBrokerAddressInPublish(name); +++ } +++ }, serviceDetector); +++ } +++ private Optional pickTopic() { +++ if (topicPublishInfoTable.isEmpty()) { +++ return Optional.absent(); +++ } +++ return Optional.of(topicPublishInfoTable.keySet().iterator().next()); +++ } ++ public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { ++ this.checkForbiddenHookList.add(checkForbiddenHook); ++ log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(), ++@@ -229,6 +263,10 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ mQClientFactory.start(); ++ } ++ +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ this.mqFaultStrategy.startDetector(); +++ } +++ ++ log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), ++ this.defaultMQProducer.isSendMessageWithVIPChannel()); ++ this.serviceState = ServiceState.RUNNING; ++@@ -273,6 +311,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ if (shutdownFactory) { ++ this.mQClientFactory.shutdown(); ++ } +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ this.mqFaultStrategy.shutdown(); +++ } ++ RequestFutureHolder.getInstance().shutdown(this); ++ log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); ++ this.serviceState = ServiceState.SHUTDOWN_ALREADY; ++@@ -574,7 +615,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ } ++ ++ public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg, ++- final long timeout) throws MQClientException, RemotingTooMuchRequestException { +++ final long timeout) throws MQClientException, RemotingTooMuchRequestException { ++ long beginStartTime = System.currentTimeMillis(); ++ this.makeSureStateOK(); ++ Validators.checkMessage(msg, this.defaultMQProducer); ++@@ -584,7 +625,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ MessageQueue mq = null; ++ try { ++ List messageQueueList = ++- mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); +++ mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); ++ Message userMessage = MessageAccessor.cloneMessage(msg); ++ String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace()); ++ userMessage.setTopic(userTopic); ++@@ -609,12 +650,13 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); ++ } ++ ++- public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { ++- return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); +++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { +++ return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName, resetIndex); ++ } ++ ++- public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { ++- this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); +++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, +++ boolean reachable) { +++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); ++ } ++ ++ private void validateNameServerSetting() throws MQClientException { ++@@ -647,9 +689,13 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; ++ int times = 0; ++ String[] brokersSent = new String[timesTotal]; +++ boolean resetIndex = false; ++ for (; times < timesTotal; times++) { ++ String lastBrokerName = null == mq ? null : mq.getBrokerName(); ++- MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); +++ if (times > 0) { +++ resetIndex = true; +++ } +++ MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName, resetIndex); ++ if (mqSelected != null) { ++ mq = mqSelected; ++ brokersSent[times] = mq.getBrokerName(); ++@@ -667,7 +713,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ ++ sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); ++ endTimestamp = System.currentTimeMillis(); ++- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); ++ switch (communicationMode) { ++ case ASYNC: ++ return null; ++@@ -684,9 +730,22 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ default: ++ break; ++ } ++- } catch (RemotingException | MQClientException e) { +++ } catch (MQClientException e) { ++ endTimestamp = System.currentTimeMillis(); ++- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); +++ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); +++ log.warn(msg.toString()); +++ exception = e; +++ continue; +++ } catch (RemotingException e) { +++ endTimestamp = System.currentTimeMillis(); +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ // Set this broker unreachable when detecting schedule task is running for RemotingException. +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false); +++ } else { +++ // Otherwise, isolate this broker. +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, true); +++ } ++ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); ++ if (log.isDebugEnabled()) { ++ log.debug(msg.toString()); ++@@ -695,7 +754,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ continue; ++ } catch (MQBrokerException e) { ++ endTimestamp = System.currentTimeMillis(); ++- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false); ++ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); ++ if (log.isDebugEnabled()) { ++ log.debug(msg.toString()); ++@@ -712,7 +771,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ } ++ } catch (InterruptedException e) { ++ endTimestamp = System.currentTimeMillis(); ++- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); ++ log.warn("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); ++ if (log.isDebugEnabled()) { ++ log.debug(msg.toString()); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java ++index 275ada7ac..37b1f3252 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java ++@@ -18,6 +18,8 @@ package org.apache.rocketmq.client.impl.producer; ++ ++ import java.util.ArrayList; ++ import java.util.List; +++ +++import com.google.common.base.Preconditions; ++ import org.apache.rocketmq.client.common.ThreadLocalIndex; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.remoting.protocol.route.QueueData; ++@@ -30,6 +32,10 @@ public class TopicPublishInfo { ++ private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex(); ++ private TopicRouteData topicRouteData; ++ +++ public interface QueueFilter { +++ boolean filter(MessageQueue mq); +++ } +++ ++ public boolean isOrderTopic() { ++ return orderTopic; ++ } ++@@ -66,6 +72,40 @@ public class TopicPublishInfo { ++ this.haveTopicRouterInfo = haveTopicRouterInfo; ++ } ++ +++ public MessageQueue selectOneMessageQueue(QueueFilter ...filter) { +++ return selectOneMessageQueue(this.messageQueueList, this.sendWhichQueue, filter); +++ } +++ +++ private MessageQueue selectOneMessageQueue(List messageQueueList, ThreadLocalIndex sendQueue, QueueFilter ...filter) { +++ if (messageQueueList == null || messageQueueList.isEmpty()) { +++ return null; +++ } +++ +++ if (filter != null && filter.length != 0) { +++ for (int i = 0; i < messageQueueList.size(); i++) { +++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); +++ MessageQueue mq = messageQueueList.get(index); +++ boolean filterResult = true; +++ for (QueueFilter f: filter) { +++ Preconditions.checkNotNull(f); +++ filterResult &= f.filter(mq); +++ } +++ if (filterResult) { +++ return mq; +++ } +++ } +++ +++ return null; +++ } +++ +++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); +++ return messageQueueList.get(index); +++ } +++ +++ public void resetIndex() { +++ this.sendWhichQueue.reset(); +++ } +++ ++ public MessageQueue selectOneMessageQueue(final String lastBrokerName) { ++ if (lastBrokerName == null) { ++ return selectOneMessageQueue(); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++index 09a8aa461..72d2f3450 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++@@ -18,11 +18,75 @@ ++ package org.apache.rocketmq.client.latency; ++ ++ public interface LatencyFaultTolerance { ++- void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration); +++ /** +++ * Update brokers' states, to decide if they are good or not. +++ * +++ * @param name Broker's name. +++ * @param currentLatency Current message sending process's latency. +++ * @param notAvailableDuration Corresponding not available time, ms. The broker will be not available until it +++ * spends such time. +++ * @param reachable To decide if this broker is reachable or not. +++ */ +++ void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration, +++ final boolean reachable); ++ +++ /** +++ * To check if this broker is available. +++ * +++ * @param name Broker's name. +++ * @return boolean variable, if this is true, then the broker is available. +++ */ ++ boolean isAvailable(final T name); ++ +++ /** +++ * To check if this broker is reachable. +++ * +++ * @param name Broker's name. +++ * @return boolean variable, if this is true, then the broker is reachable. +++ */ +++ boolean isReachable(final T name); +++ +++ /** +++ * Remove the broker in this fault item table. +++ * +++ * @param name broker's name. +++ */ ++ void remove(final T name); ++ +++ /** +++ * The worst situation, no broker can be available. Then choose random one. +++ * +++ * @return A random mq will be returned. +++ */ ++ T pickOneAtLeast(); +++ +++ /** +++ * Start a new thread, to detect the broker's reachable tag. +++ */ +++ void startDetector(); +++ +++ /** +++ * Shutdown threads that started by LatencyFaultTolerance. +++ */ +++ void shutdown(); +++ +++ /** +++ * A function reserved, just detect by once, won't create a new thread. +++ */ +++ void detectByOneRound(); +++ +++ /** +++ * Use it to set the detect timeout bound. +++ * +++ * @param detectTimeout timeout bound +++ */ +++ void setDetectTimeout(final int detectTimeout); +++ +++ /** +++ * Use it to set the detector's detector interval for each broker (each broker will be detected once during this +++ * time) +++ * +++ * @param detectInterval each broker's detecting interval +++ */ +++ void setDetectInterval(final int detectInterval); ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++index 93795d957..8af629574 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++@@ -21,30 +21,97 @@ import java.util.Collections; ++ import java.util.Enumeration; ++ import java.util.LinkedList; ++ import java.util.List; +++import java.util.Map; ++ import java.util.concurrent.ConcurrentHashMap; +++import java.util.concurrent.Executors; +++import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ThreadFactory; +++import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.client.common.ThreadLocalIndex; +++import org.apache.rocketmq.logging.org.slf4j.Logger; +++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance { ++- private final ConcurrentHashMap faultItemTable = new ConcurrentHashMap<>(16); +++ private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class); +++ private final ConcurrentHashMap faultItemTable = new ConcurrentHashMap(16); +++ private int detectTimeout = 200; +++ private int detectInterval = 2000; +++ private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex(); +++ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { +++ @Override +++ public Thread newThread(Runnable r) { +++ return new Thread(r, "LatencyFaultToleranceScheduledThread"); +++ } +++ }); ++ ++- private final ThreadLocalIndex randomItem = new ThreadLocalIndex(); +++ private final Resolver resolver; +++ +++ private final ServiceDetector serviceDetector; +++ +++ public LatencyFaultToleranceImpl(Resolver resolver, ServiceDetector serviceDetector) { +++ this.resolver = resolver; +++ this.serviceDetector = serviceDetector; +++ } +++ +++ public void detectByOneRound() { +++ for (Map.Entry item : this.faultItemTable.entrySet()) { +++ FaultItem brokerItem = item.getValue(); +++ if (System.currentTimeMillis() - brokerItem.checkStamp >= 0) { +++ brokerItem.checkStamp = System.currentTimeMillis() + this.detectInterval; +++ String brokerAddr = resolver.resolve(brokerItem.getName()); +++ if (brokerAddr == null) { +++ faultItemTable.remove(item.getKey()); +++ continue; +++ } +++ if (null == serviceDetector) { +++ continue; +++ } +++ boolean serviceOK = serviceDetector.detect(brokerAddr, detectTimeout); +++ if (serviceOK && !brokerItem.reachableFlag) { +++ log.info(brokerItem.name + " is reachable now, then it can be used."); +++ brokerItem.reachableFlag = true; +++ } +++ } +++ } +++ } +++ +++ public void startDetector() { +++ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { +++ @Override +++ public void run() { +++ try { +++ detectByOneRound(); +++ } catch (Exception e) { +++ log.warn("Unexpected exception raised while detecting service reachability", e); +++ } +++ } +++ }, 3, 3, TimeUnit.SECONDS); +++ } +++ +++ public void shutdown() { +++ this.scheduledExecutorService.shutdown(); +++ } ++ ++ @Override ++- public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration) { +++ public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration, +++ final boolean reachable) { ++ FaultItem old = this.faultItemTable.get(name); ++ if (null == old) { ++ final FaultItem faultItem = new FaultItem(name); ++ faultItem.setCurrentLatency(currentLatency); ++- faultItem.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); ++- +++ faultItem.updateNotAvailableDuration(notAvailableDuration); +++ faultItem.setReachable(reachable); ++ old = this.faultItemTable.putIfAbsent(name, faultItem); ++- if (old != null) { ++- old.setCurrentLatency(currentLatency); ++- old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); ++- } ++- } else { +++ } +++ +++ if (null != old) { ++ old.setCurrentLatency(currentLatency); ++- old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); +++ old.updateNotAvailableDuration(notAvailableDuration); +++ old.setReachable(reachable); +++ } +++ +++ if (!reachable) { +++ log.info(name + " is unreachable, it will not be used until it's reachable"); ++ } ++ } ++ ++@@ -57,6 +124,14 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ return true; ++ } ++ +++ public boolean isReachable(final String name) { +++ final FaultItem faultItem = this.faultItemTable.get(name); +++ if (faultItem != null) { +++ return faultItem.isReachable(); +++ } +++ return true; +++ } +++ ++ @Override ++ public void remove(final String name) { ++ this.faultItemTable.remove(name); ++@@ -65,68 +140,98 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ @Override ++ public String pickOneAtLeast() { ++ final Enumeration elements = this.faultItemTable.elements(); ++- List tmpList = new LinkedList<>(); +++ List tmpList = new LinkedList(); ++ while (elements.hasMoreElements()) { ++ final FaultItem faultItem = elements.nextElement(); ++ tmpList.add(faultItem); ++ } +++ ++ if (!tmpList.isEmpty()) { ++- Collections.sort(tmpList); ++- final int half = tmpList.size() / 2; ++- if (half <= 0) { ++- return tmpList.get(0).getName(); ++- } else { ++- final int i = this.randomItem.incrementAndGet() % half; ++- return tmpList.get(i).getName(); +++ Collections.shuffle(tmpList); +++ for (FaultItem faultItem : tmpList) { +++ if (faultItem.reachableFlag) { +++ return faultItem.name; +++ } ++ } ++ } +++ ++ return null; ++ } ++ ++ @Override ++ public String toString() { ++ return "LatencyFaultToleranceImpl{" + ++- "faultItemTable=" + faultItemTable + ++- ", whichItemWorst=" + randomItem + ++- '}'; +++ "faultItemTable=" + faultItemTable + +++ ", whichItemWorst=" + whichItemWorst + +++ '}'; +++ } +++ +++ public void setDetectTimeout(final int detectTimeout) { +++ this.detectTimeout = detectTimeout; ++ } ++ ++- class FaultItem implements Comparable { +++ public void setDetectInterval(final int detectInterval) { +++ this.detectInterval = detectInterval; +++ } +++ +++ public class FaultItem implements Comparable { ++ private final String name; ++ private volatile long currentLatency; ++ private volatile long startTimestamp; +++ private volatile long checkStamp; +++ private volatile boolean reachableFlag; ++ ++ public FaultItem(final String name) { ++ this.name = name; ++ } ++ +++ public void updateNotAvailableDuration(long notAvailableDuration) { +++ if (notAvailableDuration > 0 && System.currentTimeMillis() + notAvailableDuration > this.startTimestamp) { +++ this.startTimestamp = System.currentTimeMillis() + notAvailableDuration; +++ log.info(name + " will be isolated for " + notAvailableDuration + " ms."); +++ } +++ } +++ ++ @Override ++ public int compareTo(final FaultItem other) { ++ if (this.isAvailable() != other.isAvailable()) { ++- if (this.isAvailable()) +++ if (this.isAvailable()) { ++ return -1; +++ } ++ ++- if (other.isAvailable()) +++ if (other.isAvailable()) { ++ return 1; +++ } ++ } ++ ++- if (this.currentLatency < other.currentLatency) +++ if (this.currentLatency < other.currentLatency) { ++ return -1; ++- else if (this.currentLatency > other.currentLatency) { +++ } else if (this.currentLatency > other.currentLatency) { ++ return 1; ++ } ++ ++- if (this.startTimestamp < other.startTimestamp) +++ if (this.startTimestamp < other.startTimestamp) { ++ return -1; ++- else if (this.startTimestamp > other.startTimestamp) { +++ } else if (this.startTimestamp > other.startTimestamp) { ++ return 1; ++ } ++- ++ return 0; ++ } ++ +++ public void setReachable(boolean reachableFlag) { +++ this.reachableFlag = reachableFlag; +++ } +++ +++ public void setCheckStamp(long checkStamp) { +++ this.checkStamp = checkStamp; +++ } +++ ++ public boolean isAvailable() { ++- return (System.currentTimeMillis() - startTimestamp) >= 0; +++ return reachableFlag && System.currentTimeMillis() >= startTimestamp; +++ } +++ +++ public boolean isReachable() { +++ return reachableFlag; ++ } ++ ++ @Override ++@@ -139,28 +244,32 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ ++ @Override ++ public boolean equals(final Object o) { ++- if (this == o) +++ if (this == o) { ++ return true; ++- if (!(o instanceof FaultItem)) +++ } +++ if (!(o instanceof FaultItem)) { ++ return false; +++ } ++ ++ final FaultItem faultItem = (FaultItem) o; ++ ++- if (getCurrentLatency() != faultItem.getCurrentLatency()) +++ if (getCurrentLatency() != faultItem.getCurrentLatency()) { ++ return false; ++- if (getStartTimestamp() != faultItem.getStartTimestamp()) +++ } +++ if (getStartTimestamp() != faultItem.getStartTimestamp()) { ++ return false; +++ } ++ return getName() != null ? getName().equals(faultItem.getName()) : faultItem.getName() == null; ++- ++ } ++ ++ @Override ++ public String toString() { ++ return "FaultItem{" + ++- "name='" + name + '\'' + ++- ", currentLatency=" + currentLatency + ++- ", startTimestamp=" + startTimestamp + ++- '}'; +++ "name='" + name + '\'' + +++ ", currentLatency=" + currentLatency + +++ ", startTimestamp=" + startTimestamp + +++ ", reachableFlag=" + reachableFlag + +++ '}'; ++ } ++ ++ public String getName() { ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++index 1e1953fad..c01490784 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++@@ -17,25 +17,86 @@ ++ ++ package org.apache.rocketmq.client.latency; ++ ++-import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +++import org.apache.rocketmq.client.impl.producer.TopicPublishInfo.QueueFilter; ++ import org.apache.rocketmq.common.message.MessageQueue; ++-import org.apache.rocketmq.logging.org.slf4j.Logger; ++-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ public class MQFaultStrategy { ++- private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class); ++- private final LatencyFaultTolerance latencyFaultTolerance = new LatencyFaultToleranceImpl(); +++ private LatencyFaultTolerance latencyFaultTolerance; +++ private boolean sendLatencyFaultEnable; +++ private boolean startDetectorEnable; +++ private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L}; +++ private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L}; ++ ++- private boolean sendLatencyFaultEnable = false; +++ public static class BrokerFilter implements QueueFilter { +++ private String lastBrokerName; +++ +++ public void setLastBrokerName(String lastBrokerName) { +++ this.lastBrokerName = lastBrokerName; +++ } +++ +++ @Override public boolean filter(MessageQueue mq) { +++ if (lastBrokerName != null) { +++ return !mq.getBrokerName().equals(lastBrokerName); +++ } +++ return true; +++ } +++ } +++ +++ private ThreadLocal threadBrokerFilter = new ThreadLocal() { +++ @Override protected BrokerFilter initialValue() { +++ return new BrokerFilter(); +++ } +++ }; +++ +++ private QueueFilter reachableFilter = new QueueFilter() { +++ @Override public boolean filter(MessageQueue mq) { +++ return latencyFaultTolerance.isReachable(mq.getBrokerName()); +++ } +++ }; +++ +++ private QueueFilter availableFilter = new QueueFilter() { +++ @Override public boolean filter(MessageQueue mq) { +++ return latencyFaultTolerance.isAvailable(mq.getBrokerName()); +++ } +++ }; +++ +++ +++ public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) { +++ this.setStartDetectorEnable(cc.isStartDetectorEnable()); +++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); +++ this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector); +++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); +++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); +++ } +++ +++ // For unit test. +++ public MQFaultStrategy(ClientConfig cc, LatencyFaultTolerance tolerance) { +++ this.setStartDetectorEnable(cc.isStartDetectorEnable()); +++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); +++ this.latencyFaultTolerance = tolerance; +++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); +++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); +++ } ++ ++- private long[] latencyMax = {50L, 100L, 550L, 1000L, 2000L, 3000L, 15000L}; ++- private long[] notAvailableDuration = {0L, 0L, 30000L, 60000L, 120000L, 180000L, 600000L}; ++ ++ public long[] getNotAvailableDuration() { ++ return notAvailableDuration; ++ } ++ +++ public QueueFilter getAvailableFilter() { +++ return availableFilter; +++ } +++ +++ public QueueFilter getReachableFilter() { +++ return reachableFilter; +++ } +++ +++ public ThreadLocal getThreadBrokerFilter() { +++ return threadBrokerFilter; +++ } +++ ++ public void setNotAvailableDuration(final long[] notAvailableDuration) { ++ this.notAvailableDuration = notAvailableDuration; ++ } ++@@ -56,51 +117,69 @@ public class MQFaultStrategy { ++ this.sendLatencyFaultEnable = sendLatencyFaultEnable; ++ } ++ ++- public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { +++ public boolean isStartDetectorEnable() { +++ return startDetectorEnable; +++ } +++ +++ public void setStartDetectorEnable(boolean startDetectorEnable) { +++ this.startDetectorEnable = startDetectorEnable; +++ } +++ +++ public void startDetector() { +++ // user should start the detector +++ // and the thread should not be in running state. +++ if (this.sendLatencyFaultEnable && this.startDetectorEnable) { +++ // start the detector. +++ this.latencyFaultTolerance.startDetector(); +++ } +++ } +++ +++ public void shutdown() { +++ if (this.sendLatencyFaultEnable && this.startDetectorEnable) { +++ this.latencyFaultTolerance.shutdown(); +++ } +++ } +++ +++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { +++ BrokerFilter brokerFilter = threadBrokerFilter.get(); +++ brokerFilter.setLastBrokerName(lastBrokerName); ++ if (this.sendLatencyFaultEnable) { ++- try { ++- int index = tpInfo.getSendWhichQueue().incrementAndGet(); ++- for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) { ++- int pos = index++ % tpInfo.getMessageQueueList().size(); ++- MessageQueue mq = tpInfo.getMessageQueueList().get(pos); ++- if (!StringUtils.equals(lastBrokerName, mq.getBrokerName()) && latencyFaultTolerance.isAvailable(mq.getBrokerName())) { ++- return mq; ++- } ++- } ++- ++- final String notBestBroker = latencyFaultTolerance.pickOneAtLeast(); ++- int writeQueueNums = tpInfo.getWriteQueueNumsByBroker(notBestBroker); ++- if (writeQueueNums > 0) { ++- final MessageQueue mq = tpInfo.selectOneMessageQueue(); ++- if (notBestBroker != null) { ++- mq.setBrokerName(notBestBroker); ++- mq.setQueueId(tpInfo.getSendWhichQueue().incrementAndGet() % writeQueueNums); ++- } ++- return mq; ++- } else { ++- latencyFaultTolerance.remove(notBestBroker); ++- } ++- } catch (Exception e) { ++- log.error("Error occurred when selecting message queue", e); +++ if (resetIndex) { +++ tpInfo.resetIndex(); +++ } +++ MessageQueue mq = tpInfo.selectOneMessageQueue(availableFilter, brokerFilter); +++ if (mq != null) { +++ return mq; +++ } +++ +++ mq = tpInfo.selectOneMessageQueue(reachableFilter, brokerFilter); +++ if (mq != null) { +++ return mq; ++ } ++ ++ return tpInfo.selectOneMessageQueue(); ++ } ++ ++- return tpInfo.selectOneMessageQueue(lastBrokerName); +++ MessageQueue mq = tpInfo.selectOneMessageQueue(brokerFilter); +++ if (mq != null) { +++ return mq; +++ } +++ return tpInfo.selectOneMessageQueue(); ++ } ++ ++- public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { +++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, +++ final boolean reachable) { ++ if (this.sendLatencyFaultEnable) { ++- long duration = computeNotAvailableDuration(isolation ? 30000 : currentLatency); ++- this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration); +++ long duration = computeNotAvailableDuration(isolation ? 10000 : currentLatency); +++ this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration, reachable); ++ } ++ } ++ ++ private long computeNotAvailableDuration(final long currentLatency) { ++ for (int i = latencyMax.length - 1; i >= 0; i--) { ++- if (currentLatency >= latencyMax[i]) +++ if (currentLatency >= latencyMax[i]) { ++ return this.notAvailableDuration[i]; +++ } ++ } ++ ++ return 0; ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java ++similarity index 65% ++rename from remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java ++rename to client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java ++index 6aa547047..1c29ba334 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java ++@@ -14,20 +14,9 @@ ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++-package org.apache.rocketmq.remoting.protocol.body; +++package org.apache.rocketmq.client.latency; ++ ++-import java.util.ArrayList; ++-import java.util.List; ++-import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +++public interface Resolver { ++ ++-public class GetRemoteClientConfigBody extends RemotingSerializable { ++- private List keys = new ArrayList<>(); ++- ++- public List getKeys() { ++- return keys; ++- } ++- ++- public void setKeys(List keys) { ++- this.keys = keys; ++- } +++ String resolve(String name); ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java ++new file mode 100644 ++index 000000000..c6ffbad1c ++--- /dev/null +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java ++@@ -0,0 +1,30 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.client.latency; +++ +++/** +++ * Detect whether the remote service state is normal. +++ */ +++public interface ServiceDetector { +++ +++ /** +++ * Check if the remote service is normal. +++ * @param endpoint Service endpoint to check against +++ * @return true if the service is back to normal; false otherwise. +++ */ +++ boolean detect(String endpoint, long timeoutMillis); +++} ++diff --git a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java ++index 86690e40b..42ccdae5a 100644 ++--- a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java +++++ b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java ++@@ -16,11 +16,14 @@ ++ */ ++ package org.apache.rocketmq.client.latency; ++ ++-import java.util.concurrent.TimeUnit; +++import org.awaitility.core.ThrowingRunnable; ++ import org.junit.Before; ++ import org.junit.Test; ++ +++import java.util.concurrent.TimeUnit; +++ ++ import static org.assertj.core.api.Assertions.assertThat; +++import static org.awaitility.Awaitility.await; ++ ++ public class LatencyFaultToleranceImplTest { ++ private LatencyFaultTolerance latencyFaultTolerance; ++@@ -29,28 +32,31 @@ public class LatencyFaultToleranceImplTest { ++ ++ @Before ++ public void init() { ++- latencyFaultTolerance = new LatencyFaultToleranceImpl(); +++ latencyFaultTolerance = new LatencyFaultToleranceImpl(null, null); ++ } ++ ++ @Test ++ public void testUpdateFaultItem() throws Exception { ++- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000); +++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true); ++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); ++ assertThat(latencyFaultTolerance.isAvailable(anotherBrokerName)).isTrue(); ++ } ++ ++ @Test ++ public void testIsAvailable() throws Exception { ++- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50); +++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50, true); ++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); ++ ++- TimeUnit.MILLISECONDS.sleep(70); ++- assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); +++ await().atMost(500, TimeUnit.MILLISECONDS).untilAsserted(new ThrowingRunnable() { +++ @Override public void run() throws Throwable { +++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); +++ } +++ }); ++ } ++ ++ @Test ++ public void testRemove() throws Exception { ++- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000); +++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true); ++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); ++ latencyFaultTolerance.remove(brokerName); ++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); ++@@ -58,10 +64,20 @@ public class LatencyFaultToleranceImplTest { ++ ++ @Test ++ public void testPickOneAtLeast() throws Exception { ++- latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000); +++ latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true); ++ assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); ++ ++- latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000); ++- assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); +++ // Bad case, since pickOneAtLeast's behavior becomes random +++ // latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, "127.0.0.1:12011", true); +++ // assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); +++ } +++ +++ @Test +++ public void testIsReachable() throws Exception { +++ latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true); +++ assertThat(latencyFaultTolerance.isReachable(brokerName)).isEqualTo(true); +++ +++ latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, false); +++ assertThat(latencyFaultTolerance.isReachable(anotherBrokerName)).isEqualTo(false); ++ } ++ } ++\ No newline at end of file ++diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java ++index fada0efd7..485b95c42 100644 ++--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java ++@@ -41,7 +41,6 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; ++ import org.apache.rocketmq.remoting.protocol.body.GetBrokerMemberGroupResponseBody; ++-import org.apache.rocketmq.remoting.protocol.body.GetRemoteClientConfigBody; ++ import org.apache.rocketmq.remoting.protocol.body.RegisterBrokerBody; ++ import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; ++ import org.apache.rocketmq.remoting.protocol.body.TopicList; ++@@ -132,8 +131,6 @@ public class DefaultRequestProcessor implements NettyRequestProcessor { ++ return this.updateConfig(ctx, request); ++ case RequestCode.GET_NAMESRV_CONFIG: ++ return this.getConfig(ctx, request); ++- case RequestCode.GET_CLIENT_CONFIG: ++- return this.getClientConfigs(ctx, request); ++ default: ++ String error = " request type " + request.getCode() + " not supported"; ++ return RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error); ++@@ -661,25 +658,4 @@ public class DefaultRequestProcessor implements NettyRequestProcessor { ++ return response; ++ } ++ ++- private RemotingCommand getClientConfigs(ChannelHandlerContext ctx, RemotingCommand request) { ++- final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- final GetRemoteClientConfigBody body = GetRemoteClientConfigBody.decode(request.getBody(), GetRemoteClientConfigBody.class); ++- ++- String content = this.namesrvController.getConfiguration().getClientConfigsFormatString(body.getKeys()); ++- if (StringUtils.isNotBlank(content)) { ++- try { ++- response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); ++- } catch (UnsupportedEncodingException e) { ++- log.error("getConfig error, ", e); ++- response.setCode(ResponseCode.SYSTEM_ERROR); ++- response.setRemark("UnsupportedEncodingException " + e); ++- return response; ++- } ++- } ++- ++- response.setCode(ResponseCode.SUCCESS); ++- response.setRemark(null); ++- return response; ++- } ++- ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 2994893d7..b2478fec3 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -232,6 +232,12 @@ public class ProxyConfig implements ConfigFile { ++ private String remotingAccessAddr = ""; ++ private int remotingListenPort = 8080; ++ +++ // related to proxy's send strategy in cluster mode. +++ private boolean sendLatencyEnable = false; +++ private boolean startDetectorEnable = false; +++ private int detectTimeout = 200; +++ private int detectInterval = 2 * 1000; +++ ++ private int remotingHeartbeatThreadPoolNums = 2 * PROCESSOR_NUMBER; ++ private int remotingTopicRouteThreadPoolNums = 2 * PROCESSOR_NUMBER; ++ private int remotingSendMessageThreadPoolNums = 4 * PROCESSOR_NUMBER; ++@@ -1409,6 +1415,46 @@ public class ProxyConfig implements ConfigFile { ++ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; ++ } ++ +++ public boolean isSendLatencyEnable() { +++ return sendLatencyEnable; +++ } +++ +++ public boolean isStartDetectorEnable() { +++ return startDetectorEnable; +++ } +++ +++ public void setStartDetectorEnable(boolean startDetectorEnable) { +++ this.startDetectorEnable = startDetectorEnable; +++ } +++ +++ public void setSendLatencyEnable(boolean sendLatencyEnable) { +++ this.sendLatencyEnable = sendLatencyEnable; +++ } +++ +++ public boolean getStartDetectorEnable() { +++ return this.startDetectorEnable; +++ } +++ +++ public boolean getSendLatencyEnable() { +++ return this.sendLatencyEnable; +++ } +++ +++ public int getDetectTimeout() { +++ return detectTimeout; +++ } +++ +++ public void setDetectTimeout(int detectTimeout) { +++ this.detectTimeout = detectTimeout; +++ } +++ +++ public int getDetectInterval() { +++ return detectInterval; +++ } +++ +++ public void setDetectInterval(int detectInterval) { +++ this.detectInterval = detectInterval; +++ } +++ ++ public boolean isEnableBatchAck() { ++ return enableBatchAck; ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java ++index 6146c80cd..f670df205 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java ++@@ -382,7 +382,7 @@ public class SendMessageActivity extends AbstractMessingActivity { ++ int bucket = Hashing.consistentHash(shardingKey.hashCode(), writeQueues.size()); ++ targetMessageQueue = writeQueues.get(bucket); ++ } else { ++- targetMessageQueue = messageQueueView.getWriteSelector().selectOne(false); +++ targetMessageQueue = messageQueueView.getWriteSelector().selectOneByPipeline(false); ++ } ++ return targetMessageQueue; ++ } catch (Exception e) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java ++index 0d0c62168..a80f6df0b 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.processor; ++ import java.util.List; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutorService; +++ ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.client.producer.SendStatus; ++@@ -66,6 +67,8 @@ public class ProducerProcessor extends AbstractProcessor { ++ public CompletableFuture> sendMessage(ProxyContext ctx, QueueSelector queueSelector, ++ String producerGroup, int sysFlag, List messageList, long timeoutMillis) { ++ CompletableFuture> future = new CompletableFuture<>(); +++ long beginTimestampFirst = System.currentTimeMillis(); +++ AddressableMessageQueue messageQueue = null; ++ try { ++ Message message = messageList.get(0); ++ String topic = message.getTopic(); ++@@ -79,7 +82,7 @@ public class ProducerProcessor extends AbstractProcessor { ++ } ++ } ++ } ++- AddressableMessageQueue messageQueue = queueSelector.select(ctx, +++ messageQueue = queueSelector.select(ctx, ++ this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(ctx, topic)); ++ if (messageQueue == null) { ++ throw new ProxyException(ProxyExceptionCode.FORBIDDEN, "no writable queue"); ++@@ -90,6 +93,7 @@ public class ProducerProcessor extends AbstractProcessor { ++ } ++ SendMessageRequestHeader requestHeader = buildSendMessageRequestHeader(messageList, producerGroup, sysFlag, messageQueue.getQueueId()); ++ +++ AddressableMessageQueue finalMessageQueue = messageQueue; ++ future = this.serviceManager.getMessageService().sendMessage( ++ ctx, ++ messageQueue, ++@@ -102,11 +106,19 @@ public class ProducerProcessor extends AbstractProcessor { ++ if (SendStatus.SEND_OK.equals(sendResult.getSendStatus()) && ++ tranType == MessageSysFlag.TRANSACTION_PREPARED_TYPE && ++ StringUtils.isNotBlank(sendResult.getTransactionId())) { ++- fillTransactionData(ctx, producerGroup, messageQueue, sendResult, messageList); +++ fillTransactionData(ctx, producerGroup, finalMessageQueue, sendResult, messageList); ++ } ++ } ++ return sendResultList; ++- }, this.executor); +++ }, this.executor) +++ .whenComplete((result, exception) -> { +++ long endTimestamp = System.currentTimeMillis(); +++ if (exception != null) { +++ this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(), endTimestamp - beginTimestampFirst, true, false); +++ } else { +++ this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(),endTimestamp - beginTimestampFirst, false, true); +++ } +++ }); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java ++index d67b68f38..aced15cee 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java ++@@ -54,7 +54,7 @@ public class LocalTopicRouteService extends TopicRouteService { ++ @Override ++ public MessageQueueView getCurrentMessageQueueView(ProxyContext ctx, String topic) throws Exception { ++ TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic); ++- return new MessageQueueView(topic, toTopicRouteData(topicConfig)); +++ return new MessageQueueView(topic, toTopicRouteData(topicConfig), null); ++ } ++ ++ @Override ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java ++index 85cd18d45..f25fb907e 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java ++@@ -17,6 +17,7 @@ ++ package org.apache.rocketmq.proxy.service.route; ++ ++ import com.google.common.base.MoreObjects; +++import com.google.common.base.Preconditions; ++ import com.google.common.math.IntMath; ++ import java.util.ArrayList; ++ import java.util.Collections; ++@@ -30,6 +31,8 @@ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.stream.Collectors; ++ import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +++import org.apache.rocketmq.client.latency.MQFaultStrategy; ++ import org.apache.rocketmq.common.constant.PermName; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.remoting.protocol.route.QueueData; ++@@ -44,8 +47,9 @@ public class MessageQueueSelector { ++ private final Map brokerNameQueueMap = new ConcurrentHashMap<>(); ++ private final AtomicInteger queueIndex; ++ private final AtomicInteger brokerIndex; +++ private MQFaultStrategy mqFaultStrategy; ++ ++- public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, boolean read) { +++ public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, MQFaultStrategy mqFaultStrategy, boolean read) { ++ if (read) { ++ this.queues.addAll(buildRead(topicRouteWrapper)); ++ } else { ++@@ -55,6 +59,7 @@ public class MessageQueueSelector { ++ Random random = new Random(); ++ this.queueIndex = new AtomicInteger(random.nextInt()); ++ this.brokerIndex = new AtomicInteger(random.nextInt()); +++ this.mqFaultStrategy = mqFaultStrategy; ++ } ++ ++ private static List buildRead(TopicRouteWrapper topicRoute) { ++@@ -154,6 +159,86 @@ public class MessageQueueSelector { ++ return selectOneByIndex(nextIndex, onlyBroker); ++ } ++ +++ public AddressableMessageQueue selectOneByPipeline(boolean onlyBroker) { +++ if (mqFaultStrategy != null && mqFaultStrategy.isSendLatencyFaultEnable()) { +++ List messageQueueList = null; +++ MessageQueue messageQueue = null; +++ if (onlyBroker) { +++ messageQueueList = transferAddressableQueues(brokerActingQueues); +++ } else { +++ messageQueueList = transferAddressableQueues(queues); +++ } +++ AddressableMessageQueue addressableMessageQueue = null; +++ +++ // use both available filter. +++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, +++ mqFaultStrategy.getAvailableFilter(), mqFaultStrategy.getReachableFilter()); +++ addressableMessageQueue = transferQueue2Addressable(messageQueue); +++ if (addressableMessageQueue != null) { +++ return addressableMessageQueue; +++ } +++ +++ // use available filter. +++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, +++ mqFaultStrategy.getAvailableFilter()); +++ addressableMessageQueue = transferQueue2Addressable(messageQueue); +++ if (addressableMessageQueue != null) { +++ return addressableMessageQueue; +++ } +++ +++ // no available filter, then use reachable filter. +++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, +++ mqFaultStrategy.getReachableFilter()); +++ addressableMessageQueue = transferQueue2Addressable(messageQueue); +++ if (addressableMessageQueue != null) { +++ return addressableMessageQueue; +++ } +++ } +++ +++ // SendLatency is not enabled, or no queue is selected, then select by index. +++ return selectOne(onlyBroker); +++ } +++ +++ private MessageQueue selectOneMessageQueue(List messageQueueList, AtomicInteger sendQueue, TopicPublishInfo.QueueFilter...filter) { +++ if (messageQueueList == null || messageQueueList.isEmpty()) { +++ return null; +++ } +++ if (filter != null && filter.length != 0) { +++ for (int i = 0; i < messageQueueList.size(); i++) { +++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); +++ MessageQueue mq = messageQueueList.get(index); +++ boolean filterResult = true; +++ for (TopicPublishInfo.QueueFilter f: filter) { +++ Preconditions.checkNotNull(f); +++ filterResult &= f.filter(mq); +++ } +++ if (filterResult) { +++ return mq; +++ } +++ } +++ } +++ return null; +++ } +++ +++ public List transferAddressableQueues(List addressableMessageQueueList) { +++ if (addressableMessageQueueList == null) { +++ return null; +++ } +++ +++ return addressableMessageQueueList.stream() +++ .map(AddressableMessageQueue::getMessageQueue) +++ .collect(Collectors.toList()); +++ } +++ +++ private AddressableMessageQueue transferQueue2Addressable(MessageQueue messageQueue) { +++ for (AddressableMessageQueue amq: queues) { +++ if (amq.getMessageQueue().equals(messageQueue)) { +++ return amq; +++ } +++ } +++ return null; +++ } +++ ++ public AddressableMessageQueue selectNextOne(AddressableMessageQueue last) { ++ boolean onlyBroker = last.getQueueId() < 0; ++ AddressableMessageQueue newOne = last; ++@@ -190,6 +275,14 @@ public class MessageQueueSelector { ++ return brokerActingQueues; ++ } ++ +++ public MQFaultStrategy getMQFaultStrategy() { +++ return mqFaultStrategy; +++ } +++ +++ public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { +++ this.mqFaultStrategy = mqFaultStrategy; +++ } +++ ++ @Override ++ public boolean equals(Object o) { ++ if (this == o) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++index fe5387cfd..8b3c2f7c8 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++@@ -17,20 +17,22 @@ ++ package org.apache.rocketmq.proxy.service.route; ++ ++ import com.google.common.base.MoreObjects; +++import org.apache.rocketmq.client.latency.MQFaultStrategy; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++ ++ public class MessageQueueView { ++- public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData()); +++ public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData(), null); ++ ++ private final MessageQueueSelector readSelector; ++ private final MessageQueueSelector writeSelector; ++ private final TopicRouteWrapper topicRouteWrapper; +++ private MQFaultStrategy mqFaultStrategy; ++ ++- public MessageQueueView(String topic, TopicRouteData topicRouteData) { +++ public MessageQueueView(String topic, TopicRouteData topicRouteData, MQFaultStrategy mqFaultStrategy) { ++ this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic); ++ ++- this.readSelector = new MessageQueueSelector(topicRouteWrapper, true); ++- this.writeSelector = new MessageQueueSelector(topicRouteWrapper, false); +++ this.readSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, true); +++ this.writeSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, false); ++ } ++ ++ public TopicRouteData getTopicRouteData() { ++@@ -65,4 +67,12 @@ public class MessageQueueView { ++ .add("topicRouteWrapper", topicRouteWrapper) ++ .toString(); ++ } +++ +++ public MQFaultStrategy getMQFaultStrategy() { +++ return mqFaultStrategy; +++ } +++ +++ public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { +++ this.mqFaultStrategy = mqFaultStrategy; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++index 84348adc3..74769a423 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++@@ -25,7 +25,13 @@ import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; +++ +++import com.google.common.base.Optional; +++import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.exception.MQClientException; +++import org.apache.rocketmq.client.latency.MQFaultStrategy; +++import org.apache.rocketmq.client.latency.Resolver; +++import org.apache.rocketmq.client.latency.ServiceDetector; ++ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++@@ -39,6 +45,7 @@ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; +++import org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++ import org.checkerframework.checker.nullness.qual.NonNull; ++ import org.checkerframework.checker.nullness.qual.Nullable; ++@@ -47,6 +54,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); ++ ++ private final MQClientAPIFactory mqClientAPIFactory; +++ private MQFaultStrategy mqFaultStrategy; ++ ++ protected final LoadingCache topicCache; ++ protected final ScheduledExecutorService scheduledExecutorService; ++@@ -97,15 +105,83 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ } ++ } ++ }); ++- +++ ServiceDetector serviceDetector = new ServiceDetector() { +++ @Override +++ public boolean detect(String endpoint, long timeoutMillis) { +++ Optional candidateTopic = pickTopic(); +++ if (!candidateTopic.isPresent()) { +++ return false; +++ } +++ try { +++ GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); +++ requestHeader.setTopic(candidateTopic.get()); +++ requestHeader.setQueueId(0); +++ Long maxOffset = mqClientAPIFactory.getClient().getMaxOffset(endpoint, requestHeader, timeoutMillis).get(); +++ return true; +++ } catch (Exception e) { +++ return false; +++ } +++ } +++ }; +++ mqFaultStrategy = new MQFaultStrategy(extractClientConfigFromProxyConfig(config), new Resolver() { +++ @Override +++ public String resolve(String name) { +++ try { +++ String brokerAddr = getBrokerAddr(null, name); +++ return brokerAddr; +++ } catch (Exception e) { +++ return null; +++ } +++ } +++ }, serviceDetector); ++ this.init(); ++ } ++ +++ // pickup one topic in the topic cache +++ private Optional pickTopic() { +++ if (topicCache.asMap().isEmpty()) { +++ return Optional.absent(); +++ } +++ return Optional.of(topicCache.asMap().keySet().iterator().next()); +++ } +++ ++ protected void init() { ++ this.appendShutdown(this.scheduledExecutorService::shutdown); ++ this.appendStartAndShutdown(this.mqClientAPIFactory); ++ } ++ +++ @Override +++ public void shutdown() throws Exception { +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ mqFaultStrategy.shutdown(); +++ } +++ } +++ +++ @Override +++ public void start() throws Exception { +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ this.mqFaultStrategy.startDetector(); +++ } +++ } +++ +++ public ClientConfig extractClientConfigFromProxyConfig(ProxyConfig proxyConfig) { +++ ClientConfig tempClientConfig = new ClientConfig(); +++ tempClientConfig.setSendLatencyEnable(proxyConfig.getSendLatencyEnable()); +++ tempClientConfig.setStartDetectorEnable(proxyConfig.getStartDetectorEnable()); +++ tempClientConfig.setDetectTimeout(proxyConfig.getDetectTimeout()); +++ tempClientConfig.setDetectInterval(proxyConfig.getDetectInterval()); +++ return tempClientConfig; +++ } +++ +++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, +++ boolean reachable) { +++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); +++ } +++ +++ public MQFaultStrategy getMqFaultStrategy() { +++ return this.mqFaultStrategy; +++ } +++ ++ public MessageQueueView getAllMessageQueueView(ProxyContext ctx, String topicName) throws Exception { ++ return getCacheMessageQueueWrapper(this.topicCache, topicName); ++ } ++@@ -136,7 +212,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ ++ protected MessageQueueView buildMessageQueueView(String topic, TopicRouteData topicRouteData) { ++ if (isTopicRouteValid(topicRouteData)) { ++- MessageQueueView tmp = new MessageQueueView(topic, topicRouteData); +++ MessageQueueView tmp = new MessageQueueView(topic, topicRouteData, TopicRouteService.this.getMqFaultStrategy()); ++ log.debug("load topic route from namesrv. topic: {}, queue: {}", topic, tmp); ++ return tmp; ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java ++index 7fd9a9ffd..77ae5e4d1 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java ++@@ -93,7 +93,6 @@ public class ReceiveMessageActivityTest extends BaseActivityTest { ++ pollTimeCaptor.capture(), anyInt(), any(), anyBoolean(), any(), isNull(), anyLong())) ++ .thenReturn(CompletableFuture.completedFuture(new PopResult(PopStatus.NO_NEW_MSG, Collections.emptyList()))); ++ ++- ++ ProxyContext context = createContext(); ++ context.setRemainingMs(1L); ++ this.receiveMessageActivity.receiveMessage( ++@@ -274,7 +273,7 @@ public class ReceiveMessageActivityTest extends BaseActivityTest { ++ } ++ ++ @Test ++- public void testReceiveMessageQueueSelector() { +++ public void testReceiveMessageQueueSelector() throws Exception { ++ TopicRouteData topicRouteData = new TopicRouteData(); ++ List queueDatas = new ArrayList<>(); ++ for (int i = 0; i < 2; i++) { ++@@ -298,7 +297,7 @@ public class ReceiveMessageActivityTest extends BaseActivityTest { ++ } ++ topicRouteData.setBrokerDatas(brokerDatas); ++ ++- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); ++ ReceiveMessageActivity.ReceiveMessageQueueSelector selector = new ReceiveMessageActivity.ReceiveMessageQueueSelector(""); ++ ++ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java ++index 588423bb9..4882a5ed8 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java ++@@ -35,6 +35,8 @@ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.client.ClientConfig; +++import org.apache.rocketmq.client.latency.MQFaultStrategy; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.client.producer.SendStatus; ++ import org.apache.rocketmq.common.MixAll; ++@@ -49,6 +51,7 @@ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.MessageQueueView; +++import org.apache.rocketmq.proxy.service.route.TopicRouteService; ++ import org.apache.rocketmq.remoting.protocol.route.BrokerData; ++ import org.apache.rocketmq.remoting.protocol.route.QueueData; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++@@ -62,15 +65,19 @@ import static org.junit.Assert.assertThrows; ++ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyInt; ++ import static org.mockito.ArgumentMatchers.anyString; +++import static org.mockito.Mockito.mock; ++ import static org.mockito.Mockito.when; ++ ++ public class SendMessageActivityTest extends BaseActivityTest { ++ ++ protected static final String BROKER_NAME = "broker"; +++ protected static final String BROKER_NAME2 = "broker2"; ++ protected static final String CLUSTER_NAME = "cluster"; ++ protected static final String BROKER_ADDR = "127.0.0.1:10911"; +++ protected static final String BROKER_ADDR2 = "127.0.0.1:10912"; ++ private static final String TOPIC = "topic"; ++ private static final String CONSUMER_GROUP = "consumerGroup"; +++ MQFaultStrategy mqFaultStrategy; ++ ++ private SendMessageActivity sendMessageActivity; ++ ++@@ -262,7 +269,7 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ } ++ ++ @Test ++- public void testSendOrderMessageQueueSelector() { +++ public void testSendOrderMessageQueueSelector() throws Exception { ++ TopicRouteData topicRouteData = new TopicRouteData(); ++ QueueData queueData = new QueueData(); ++ BrokerData brokerData = new BrokerData(); ++@@ -277,7 +284,7 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ brokerData.setBrokerAddrs(brokerAddrs); ++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); ++ ++- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); ++ SendMessageActivity.SendMessageQueueSelector selector1 = new SendMessageActivity.SendMessageQueueSelector( ++ SendMessageRequest.newBuilder() ++ .addMessages(Message.newBuilder() ++@@ -288,6 +295,12 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ .build() ++ ); ++ +++ TopicRouteService topicRouteService = mock(TopicRouteService.class); +++ MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class); +++ when(topicRouteService.getAllMessageQueueView(any(), any())).thenReturn(messageQueueView); +++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); +++ when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false); +++ ++ SendMessageActivity.SendMessageQueueSelector selector2 = new SendMessageActivity.SendMessageQueueSelector( ++ SendMessageRequest.newBuilder() ++ .addMessages(Message.newBuilder() ++@@ -328,12 +341,17 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ brokerData.setBrokerAddrs(brokerAddrs); ++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); ++ ++- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); +++ ++ SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector( ++ SendMessageRequest.newBuilder() ++ .addMessages(Message.newBuilder().build()) ++ .build() ++ ); +++ TopicRouteService topicRouteService = mock(TopicRouteService.class); +++ MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class); +++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); +++ when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy()); ++ ++ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); ++ AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); ++@@ -343,6 +361,45 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ assertNotEquals(firstSelect, secondSelect); ++ } ++ +++ @Test +++ public void testSendNormalMessageQueueSelectorPipeLine() throws Exception { +++ TopicRouteData topicRouteData = new TopicRouteData(); +++ int queueNums = 2; +++ +++ QueueData queueData = createQueueData(BROKER_NAME, queueNums); +++ QueueData queueData2 = createQueueData(BROKER_NAME2, queueNums); +++ topicRouteData.setQueueDatas(Lists.newArrayList(queueData,queueData2)); +++ +++ +++ BrokerData brokerData = createBrokerData(CLUSTER_NAME, BROKER_NAME, BROKER_ADDR); +++ BrokerData brokerData2 = createBrokerData(CLUSTER_NAME, BROKER_NAME2, BROKER_ADDR2); +++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData, brokerData2)); +++ +++ SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector( +++ SendMessageRequest.newBuilder() +++ .addMessages(Message.newBuilder().build()) +++ .build() +++ ); +++ +++ ClientConfig cc = new ClientConfig(); +++ this.mqFaultStrategy = new MQFaultStrategy(cc, null, null); +++ mqFaultStrategy.setSendLatencyFaultEnable(true); +++ mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, true); +++ mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, false); +++ +++ TopicRouteService topicRouteService = mock(TopicRouteService.class); +++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy()); +++ +++ +++ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); +++ assertEquals(firstSelect.getBrokerName(), BROKER_NAME2); +++ +++ mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, false); +++ mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, true); +++ AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); +++ assertEquals(secondSelect.getBrokerName(), BROKER_NAME); +++ } ++ @Test ++ public void testParameterValidate() { ++ // too large message body ++@@ -850,4 +907,23 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ } ++ return sb.toString(); ++ } +++ +++ private static QueueData createQueueData(String brokerName, int writeQueueNums) { +++ QueueData queueData = new QueueData(); +++ queueData.setBrokerName(brokerName); +++ queueData.setWriteQueueNums(writeQueueNums); +++ queueData.setPerm(PermName.PERM_WRITE); +++ return queueData; +++ } +++ +++ private static BrokerData createBrokerData(String clusterName, String brokerName, String brokerAddrs) { +++ BrokerData brokerData = new BrokerData(); +++ brokerData.setCluster(clusterName); +++ brokerData.setBrokerName(brokerName); +++ HashMap brokerAddrsMap = new HashMap<>(); +++ brokerAddrsMap.put(MixAll.MASTER_ID, brokerAddrs); +++ brokerData.setBrokerAddrs(brokerAddrsMap); +++ +++ return brokerData; +++ } ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java ++index c97bd5a72..ca6fe909e 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java ++@@ -78,7 +78,7 @@ public class BaseServiceTest extends InitConfigTest { ++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); ++ ++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(ERR_TOPIC))).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "")); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData)); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData, null)); ++ } ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java ++index e44ed28f4..d150f87c4 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java ++@@ -30,12 +30,12 @@ public class MessageQueueSelectorTest extends BaseServiceTest { ++ public void testReadMessageQueue() { ++ queueData.setPerm(PermName.PERM_READ); ++ queueData.setReadQueueNums(0); ++- MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); +++ MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true); ++ assertTrue(messageQueueSelector.getQueues().isEmpty()); ++ ++ queueData.setPerm(PermName.PERM_READ); ++ queueData.setReadQueueNums(3); ++- messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); +++ messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true); ++ assertEquals(3, messageQueueSelector.getQueues().size()); ++ assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); ++ for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { ++@@ -58,12 +58,12 @@ public class MessageQueueSelectorTest extends BaseServiceTest { ++ public void testWriteMessageQueue() { ++ queueData.setPerm(PermName.PERM_WRITE); ++ queueData.setReadQueueNums(0); ++- MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); +++ MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false); ++ assertTrue(messageQueueSelector.getQueues().isEmpty()); ++ ++ queueData.setPerm(PermName.PERM_WRITE); ++ queueData.setWriteQueueNums(3); ++- messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); +++ messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false); ++ assertEquals(3, messageQueueSelector.getQueues().size()); ++ assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); ++ for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++index c67f4953d..43fba3d03 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++@@ -132,7 +132,7 @@ public class HeartbeatSyncerTest extends InitConfigTest { ++ brokerAddr.put(0L, "127.0.0.1:10911"); ++ brokerData.setBrokerAddrs(brokerAddr); ++ topicRouteData.getBrokerDatas().add(brokerData); ++- MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData); +++ MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData, null); ++ when(this.topicRouteService.getAllMessageQueueView(any(), anyString())).thenReturn(messageQueueView); ++ } ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java ++index a0063544e..91af74cbe 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java ++@@ -64,7 +64,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { ++ this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager, ++ this.mqClientAPIFactory); ++ ++- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); ++ when(this.topicRouteService.getAllMessageQueueView(any(), anyString())) ++ .thenReturn(messageQueueView); ++ ++@@ -127,7 +127,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { ++ brokerData.setBrokerAddrs(brokerAddrs); ++ topicRouteData.getQueueDatas().add(queueData); ++ topicRouteData.getBrokerDatas().add(brokerData); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null)); ++ ++ TopicRouteData clusterTopicRouteData = new TopicRouteData(); ++ QueueData clusterQueueData = new QueueData(); ++@@ -141,7 +141,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { ++ brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR); ++ clusterBrokerData.setBrokerAddrs(brokerAddrs); ++ clusterTopicRouteData.setBrokerDatas(Lists.newArrayList(clusterBrokerData)); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData, null)); ++ ++ TopicRouteData clusterTopicRouteData2 = new TopicRouteData(); ++ QueueData clusterQueueData2 = new QueueData(); ++@@ -155,7 +155,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { ++ brokerAddrs.put(MixAll.MASTER_ID, brokerAddr2); ++ clusterBrokerData2.setBrokerAddrs(brokerAddrs); ++ clusterTopicRouteData2.setBrokerDatas(Lists.newArrayList(clusterBrokerData2)); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2, null)); ++ ++ ConfigurationManager.getProxyConfig().setTransactionHeartbeatBatchNum(2); ++ this.clusterTransactionService.start(); ++-- ++2.32.0.windows.2 ++ +diff --git a/patch015-backport-fix-some-bugs.patch b/patch015-backport-fix-some-bugs.patch +new file mode 100644 +index 000000000..11c10a6bd +--- /dev/null ++++ b/patch015-backport-fix-some-bugs.patch +@@ -0,0 +1,1894 @@ ++From bd0e9c09db9748f7f74a0c707579142dccf30afc Mon Sep 17 00:00:00 2001 ++From: PiteXChen <44110731+RapperCL@users.noreply.github.com> ++Date: Tue, 29 Aug 2023 19:39:27 +0800 ++Subject: [PATCH 1/7] [ISSUE #7111] Remove responseFuture from the ++ responseTable when exception occurs (#7112) ++ ++* remove responseFuture when exception ++* Empty-Commit ++ ++--------- ++Co-authored-by: chenyong152 ++--- ++ .../apache/rocketmq/remoting/netty/NettyRemotingAbstract.java | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++index 44d6a3df4..fce2de267 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++@@ -529,6 +529,7 @@ public abstract class NettyRemotingAbstract { ++ log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); ++ }); ++ } catch (Exception e) { +++ responseTable.remove(opaque); ++ responseFuture.release(); ++ log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); ++ throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); ++-- ++2.32.0.windows.2 ++ ++ ++From c78061bf6ca5f35452510ec4107c46735c51c316 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 30 Aug 2023 22:29:51 +0800 ++Subject: [PATCH 2/7] [ISSUE#7280] Fix and refactor handle commit exception in ++ tiered storage (#7281) ++ ++* refactor handle commit exception ++ ++* refactor handle commit exception ++ ++* fix handle commit exception ++--- ++ .../tieredstore/TieredDispatcher.java | 3 +- ++ .../tieredstore/TieredMessageFetcher.java | 57 ++-- ++ .../tieredstore/TieredMessageStore.java | 26 +- ++ .../provider/TieredFileSegment.java | 291 ++++++++++-------- ++ .../provider/TieredStoreProvider.java | 8 +- ++ .../provider/posix/PosixFileSegment.java | 4 +- ++ .../CommitLogInputStream.java} | 30 +- ++ .../FileSegmentInputStream.java} | 49 ++- ++ .../FileSegmentInputStreamFactory.java} | 26 +- ++ .../tieredstore/TieredMessageStoreTest.java | 14 +- ++ .../tieredstore/file/TieredFlatFileTest.java | 2 + ++ .../tieredstore/file/TieredIndexFileTest.java | 2 + ++ ...m.java => MockFileSegmentInputStream.java} | 8 +- ++ .../TieredFileSegmentInputStreamTest.java | 24 +- ++ .../provider/TieredFileSegmentTest.java | 89 +++++- ++ .../provider/memory/MemoryFileSegment.java | 27 +- ++ .../memory/MemoryFileSegmentWithoutCheck.java | 4 +- ++ 17 files changed, 427 insertions(+), 237 deletions(-) ++ rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredCommitLogInputStream.java => stream/CommitLogInputStream.java} (88%) ++ rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredFileSegmentInputStream.java => stream/FileSegmentInputStream.java} (77%) ++ rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredFileSegmentInputStreamFactory.java => stream/FileSegmentInputStreamFactory.java} (54%) ++ rename tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/{MockTieredFileSegmentInputStream.java => MockFileSegmentInputStream.java} (82%) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++index 1746190cd..430c2b62e 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++@@ -318,8 +318,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ continue; ++ case FILE_CLOSED: ++ tieredFlatFileManager.destroyCompositeFile(flatFile.getMessageQueue()); ++- logger.info("TieredDispatcher#dispatchFlatFile: file has been close and destroy, " + ++- "topic: {}, queueId: {}", topic, queueId); +++ logger.info("File has been closed and destroy, topic: {}, queueId: {}", topic, queueId); ++ return; ++ default: ++ dispatchOffset--; ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++index 9a9a3e5a5..766ff64f6 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++@@ -273,15 +273,17 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ TieredStoreMetricsManager.cacheHit.add(resultWrapperList.size(), attributes); ++ } ++ ++- // if no cached message found and there is currently an inflight request, wait for the request to end before continuing +++ // If there are no messages in the cache and there are currently requests being pulled. +++ // We need to wait for the request to return before continuing. ++ if (resultWrapperList.isEmpty() && waitInflightRequest) { ++- CompletableFuture future = flatFile.getInflightRequest(group, queueOffset, maxCount) ++- .getFuture(queueOffset); +++ CompletableFuture future = +++ flatFile.getInflightRequest(group, queueOffset, maxCount).getFuture(queueOffset); ++ if (!future.isDone()) { ++ Stopwatch stopwatch = Stopwatch.createStarted(); ++ // to prevent starvation issues, only allow waiting for inflight request once ++ return future.thenCompose(v -> { ++- LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: wait for inflight request cost: {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); +++ LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: wait for response cost: {}ms", +++ stopwatch.elapsed(TimeUnit.MILLISECONDS)); ++ return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, false); ++ }); ++ } ++@@ -302,7 +304,8 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ ++ // if cache hit, result will be returned immediately and asynchronously prefetch messages for later requests ++ if (!resultWrapperList.isEmpty()) { ++- LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: cache hit: topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}", +++ LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: cache hit: " + +++ "topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}", ++ mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, resultWrapperList.size()); ++ prefetchMessage(flatFile, group, maxCount, lastGetOffset + 1); ++ ++@@ -316,8 +319,10 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ } ++ ++ // if cache is miss, immediately pull messages ++- LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: topic: {}, queue: {}, queue offset: {}, max message num: {}", +++ LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + +++ "topic: {}, queue: {}, queue offset: {}, max message num: {}", ++ mq.getTopic(), mq.getQueueId(), queueOffset, maxCount); +++ ++ CompletableFuture resultFuture; ++ synchronized (flatFile) { ++ int batchSize = maxCount * storeConfig.getReadAheadMinFactor(); ++@@ -453,42 +458,42 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ public CompletableFuture getMessageAsync( ++ String group, String topic, int queueId, long queueOffset, int maxCount, final MessageFilter messageFilter) { ++ +++ GetMessageResult result = new GetMessageResult(); ++ CompositeQueueFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId)); +++ ++ if (flatFile == null) { ++- GetMessageResult result = new GetMessageResult(); ++ result.setNextBeginOffset(queueOffset); ++ result.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE); ++ return CompletableFuture.completedFuture(result); ++ } ++ ++- GetMessageResult result = new GetMessageResult(); ++- long minQueueOffset = flatFile.getConsumeQueueMinOffset(); ++- long maxQueueOffset = flatFile.getConsumeQueueCommitOffset(); ++- result.setMinOffset(minQueueOffset); ++- result.setMaxOffset(maxQueueOffset); +++ // Max queue offset means next message put position +++ result.setMinOffset(flatFile.getConsumeQueueMinOffset()); +++ result.setMaxOffset(flatFile.getConsumeQueueCommitOffset()); +++ +++ // Fill result according file offset. +++ // Offset range | Result | Fix to +++ // (-oo, 0] | no message | current offset +++ // (0, min) | too small | min offset +++ // [min, max) | correct | +++ // [max, max] | overflow one | max offset +++ // (max, +oo) | overflow badly | max offset ++ ++- if (flatFile.getConsumeQueueCommitOffset() <= 0) { +++ if (result.getMaxOffset() <= 0) { ++ result.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE); ++ result.setNextBeginOffset(queueOffset); ++ return CompletableFuture.completedFuture(result); ++- } ++- ++- // request range | result ++- // (0, min) | too small ++- // [min, max) | correct ++- // [max, max] | overflow one ++- // (max, +oo) | overflow badly ++- if (queueOffset < minQueueOffset) { +++ } else if (queueOffset < result.getMinOffset()) { ++ result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL); ++- result.setNextBeginOffset(flatFile.getConsumeQueueMinOffset()); +++ result.setNextBeginOffset(result.getMinOffset()); ++ return CompletableFuture.completedFuture(result); ++- } else if (queueOffset == maxQueueOffset) { +++ } else if (queueOffset == result.getMaxOffset()) { ++ result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE); ++- result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset()); +++ result.setNextBeginOffset(result.getMaxOffset()); ++ return CompletableFuture.completedFuture(result); ++- } else if (queueOffset > maxQueueOffset) { +++ } else if (queueOffset > result.getMaxOffset()) { ++ result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY); ++- result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset()); +++ result.setNextBeginOffset(result.getMaxOffset()); ++ return CompletableFuture.completedFuture(result); ++ } ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index 5240ac8e9..78e855f36 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -99,11 +99,11 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ return storeConfig; ++ } ++ ++- public boolean viaTieredStorage(String topic, int queueId, long offset) { ++- return viaTieredStorage(topic, queueId, offset, 1); +++ public boolean fetchFromCurrentStore(String topic, int queueId, long offset) { +++ return fetchFromCurrentStore(topic, queueId, offset, 1); ++ } ++ ++- public boolean viaTieredStorage(String topic, int queueId, long offset, int batchSize) { +++ public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int batchSize) { ++ TieredMessageStoreConfig.TieredStorageLevel deepStorageLevel = storeConfig.getTieredStorageLevel(); ++ ++ if (deepStorageLevel.check(TieredMessageStoreConfig.TieredStorageLevel.FORCE)) { ++@@ -146,8 +146,10 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ public CompletableFuture getMessageAsync(String group, String topic, ++ int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { ++ ++- if (!viaTieredStorage(topic, queueId, offset, maxMsgNums)) { ++- logger.trace("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); +++ if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) { +++ logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); +++ } else { +++ logger.trace("GetMessageAsync from next store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); ++ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); ++ } ++ ++@@ -168,14 +170,14 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ ++ if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) { ++ TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes); ++- logger.debug("GetMessageAsync not found then try back to next store, result: {}, " + +++ logger.debug("GetMessageAsync not found, then back to next store, result: {}, " + ++ "topic: {}, queue: {}, queue offset: {}, offset range: {}-{}", ++ result.getStatus(), topic, queueId, offset, result.getMinOffset(), result.getMaxOffset()); ++ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); ++ } ++ } ++ ++- // system topic +++ // Fetch system topic data from the broker when using the force level. ++ if (result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { ++ if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { ++ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); ++@@ -198,7 +200,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ TieredStoreMetricsManager.messagesOutTotal.add(result.getMessageCount(), messagesOutAttributes); ++ } ++ ++- // fix min or max offset according next store +++ // Fix min or max offset according next store at last ++ long minOffsetInQueue = next.getMinOffsetInQueue(topic, queueId); ++ if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) { ++ result.setMinOffset(minOffsetInQueue); ++@@ -209,7 +211,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ } ++ return result; ++ }).exceptionally(e -> { ++- logger.error("GetMessageAsync from tiered store failed: ", e); +++ logger.error("GetMessageAsync from tiered store failed", e); ++ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); ++ }); ++ } ++@@ -251,7 +253,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ .build(); ++ TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); ++ if (time < 0) { ++- logger.debug("TieredMessageStore#getEarliestMessageTimeAsync: get earliest message time failed, try to get earliest message time from next store: topic: {}, queue: {}", +++ logger.debug("GetEarliestMessageTimeAsync failed, try to get earliest message time from next store: topic: {}, queue: {}", ++ topic, queueId); ++ return finalNextEarliestMessageTime != Long.MAX_VALUE ? finalNextEarliestMessageTime : -1; ++ } ++@@ -262,7 +264,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ @Override ++ public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, ++ long consumeQueueOffset) { ++- if (viaTieredStorage(topic, queueId, consumeQueueOffset)) { +++ if (fetchFromCurrentStore(topic, queueId, consumeQueueOffset)) { ++ Stopwatch stopwatch = Stopwatch.createStarted(); ++ return fetcher.getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset) ++ .thenApply(time -> { ++@@ -272,7 +274,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ .build(); ++ TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); ++ if (time == -1) { ++- logger.debug("TieredMessageStore#getMessageStoreTimeStampAsync: get message time failed, try to get message time from next store: topic: {}, queue: {}, queue offset: {}", +++ logger.debug("GetEarliestMessageTimeAsync failed, try to get message time from next store, topic: {}, queue: {}, queue offset: {}", ++ topic, queueId, consumeQueueOffset); ++ return next.getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset); ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java ++index 5062c7d9e..32911a6e8 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java ++@@ -16,14 +16,11 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.provider; ++ ++-import com.google.common.annotations.VisibleForTesting; ++-import com.google.common.base.Stopwatch; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; ++ import java.util.List; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.Semaphore; ++-import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.locks.ReentrantLock; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++@@ -35,8 +32,8 @@ import org.apache.rocketmq.tieredstore.exception.TieredStoreException; ++ import org.apache.rocketmq.tieredstore.file.TieredCommitLog; ++ import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue; ++ import org.apache.rocketmq.tieredstore.file.TieredIndexFile; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ ++@@ -50,22 +47,23 @@ public abstract class TieredFileSegment implements Comparable ++ protected final TieredMessageStoreConfig storeConfig; ++ ++ private final long maxSize; ++- private final ReentrantLock bufferLock; ++- private final Semaphore commitLock; +++ private final ReentrantLock bufferLock = new ReentrantLock(); +++ private final Semaphore commitLock = new Semaphore(1); ++ ++- private volatile boolean full; ++- private volatile boolean closed; +++ private volatile boolean full = false; +++ private volatile boolean closed = false; ++ ++- private volatile long minTimestamp; ++- private volatile long maxTimestamp; ++- private volatile long commitPosition; ++- private volatile long appendPosition; +++ private volatile long minTimestamp = Long.MAX_VALUE; +++ private volatile long maxTimestamp = Long.MAX_VALUE; +++ private volatile long commitPosition = 0L; +++ private volatile long appendPosition = 0L; ++ ++ // only used in commitLog ++- private volatile long dispatchCommitOffset = 0; +++ private volatile long dispatchCommitOffset = 0L; ++ ++ private ByteBuffer codaBuffer; ++- private List uploadBufferList = new ArrayList<>(); +++ private List bufferList = new ArrayList<>(); +++ private FileSegmentInputStream fileSegmentInputStream; ++ private CompletableFuture flightCommitRequest = CompletableFuture.completedFuture(false); ++ ++ public TieredFileSegment(TieredMessageStoreConfig storeConfig, ++@@ -75,21 +73,13 @@ public abstract class TieredFileSegment implements Comparable ++ this.fileType = fileType; ++ this.filePath = filePath; ++ this.baseOffset = baseOffset; ++- ++- this.closed = false; ++- this.bufferLock = new ReentrantLock(); ++- this.commitLock = new Semaphore(1); ++- ++- this.commitPosition = 0L; ++- this.appendPosition = 0L; ++- this.minTimestamp = Long.MAX_VALUE; ++- this.maxTimestamp = Long.MAX_VALUE; ++- ++- // The max segment size of a file is determined by the file type ++- this.maxSize = getMaxSizeAccordingFileType(storeConfig); +++ this.maxSize = getMaxSizeByFileType(); ++ } ++ ++- private long getMaxSizeAccordingFileType(TieredMessageStoreConfig storeConfig) { +++ /** +++ * The max segment size of a file is determined by the file type +++ */ +++ protected long getMaxSizeByFileType() { ++ switch (fileType) { ++ case COMMIT_LOG: ++ return storeConfig.getTieredStoreCommitLogMaxSize(); ++@@ -184,39 +174,23 @@ public abstract class TieredFileSegment implements Comparable ++ this.appendPosition = pos; ++ } ++ ++- private List rollingUploadBuffer() { +++ private List borrowBuffer() { ++ bufferLock.lock(); ++ try { ++- List tmp = uploadBufferList; ++- uploadBufferList = new ArrayList<>(); +++ List tmp = bufferList; +++ bufferList = new ArrayList<>(); ++ return tmp; ++ } finally { ++ bufferLock.unlock(); ++ } ++ } ++ ++- private void sendBackBuffer(TieredFileSegmentInputStream inputStream) { ++- bufferLock.lock(); ++- try { ++- List tmpBufferList = inputStream.getUploadBufferList(); ++- for (ByteBuffer buffer : tmpBufferList) { ++- buffer.rewind(); ++- } ++- tmpBufferList.addAll(uploadBufferList); ++- uploadBufferList = tmpBufferList; ++- if (inputStream.getCodaBuffer() != null) { ++- codaBuffer.rewind(); ++- } ++- } finally { ++- bufferLock.unlock(); ++- } ++- } ++- ++ @SuppressWarnings("NonAtomicOperationOnVolatileField") ++- public AppendResult append(ByteBuffer byteBuf, long timeStamp) { +++ public AppendResult append(ByteBuffer byteBuf, long timestamp) { ++ if (closed) { ++ return AppendResult.FILE_CLOSED; ++ } +++ ++ bufferLock.lock(); ++ try { ++ if (full || codaBuffer != null) { ++@@ -227,7 +201,8 @@ public abstract class TieredFileSegment implements Comparable ++ minTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_BEGIN_TIME_STAMP_POSITION); ++ maxTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_END_TIME_STAMP_POSITION); ++ appendPosition += byteBuf.remaining(); ++- uploadBufferList.add(byteBuf); +++ // IndexFile is large and not change after compaction, no need deep copy +++ bufferList.add(byteBuf); ++ setFull(); ++ return AppendResult.SUCCESS; ++ } ++@@ -236,23 +211,34 @@ public abstract class TieredFileSegment implements Comparable ++ setFull(); ++ return AppendResult.FILE_FULL; ++ } ++- if (uploadBufferList.size() > storeConfig.getTieredStoreGroupCommitCount() +++ +++ if (bufferList.size() > storeConfig.getTieredStoreGroupCommitCount() ++ || appendPosition - commitPosition > storeConfig.getTieredStoreGroupCommitSize()) { ++ commitAsync(); ++ } ++- if (uploadBufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) { ++- logger.debug("TieredFileSegment#append: buffer full: file: {}, upload buffer size: {}", ++- getPath(), uploadBufferList.size()); +++ +++ if (bufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) { +++ logger.debug("File segment append buffer full, file: {}, buffer size: {}, pending bytes: {}", +++ getPath(), bufferList.size(), appendPosition - commitPosition); ++ return AppendResult.BUFFER_FULL; ++ } ++- if (timeStamp != Long.MAX_VALUE) { ++- maxTimestamp = timeStamp; +++ +++ if (timestamp != Long.MAX_VALUE) { +++ maxTimestamp = timestamp; ++ if (minTimestamp == Long.MAX_VALUE) { ++- minTimestamp = timeStamp; +++ minTimestamp = timestamp; ++ } ++ } +++ ++ appendPosition += byteBuf.remaining(); ++- uploadBufferList.add(byteBuf); +++ +++ // deep copy buffer +++ ByteBuffer byteBuffer = ByteBuffer.allocateDirect(byteBuf.remaining()); +++ byteBuffer.put(byteBuf); +++ byteBuffer.flip(); +++ byteBuf.rewind(); +++ +++ bufferList.add(byteBuffer); ++ return AppendResult.SUCCESS; ++ } finally { ++ bufferLock.unlock(); ++@@ -267,7 +253,6 @@ public abstract class TieredFileSegment implements Comparable ++ return appendPosition; ++ } ++ ++- @VisibleForTesting ++ public void setAppendPosition(long appendPosition) { ++ this.appendPosition = appendPosition; ++ } ++@@ -333,6 +318,8 @@ public abstract class TieredFileSegment implements Comparable ++ if (closed) { ++ return false; ++ } +++ // result is false when we send real commit request +++ // use join for wait flight request done ++ Boolean result = commitAsync().join(); ++ if (!result) { ++ result = flightCommitRequest.join(); ++@@ -340,92 +327,156 @@ public abstract class TieredFileSegment implements Comparable ++ return result; ++ } ++ +++ private void releaseCommitLock() { +++ if (commitLock.availablePermits() == 0) { +++ commitLock.release(); +++ } else { +++ logger.error("[Bug] FileSegmentCommitAsync, lock is already released: available permits: {}", +++ commitLock.availablePermits()); +++ } +++ } +++ +++ private void updateDispatchCommitOffset(List bufferList) { +++ if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) { +++ dispatchCommitOffset = +++ MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1)); +++ } +++ } +++ +++ /** +++ * @return false: commit, true: no commit operation +++ */ ++ @SuppressWarnings("NonAtomicOperationOnVolatileField") ++ public CompletableFuture commitAsync() { ++ if (closed) { ++ return CompletableFuture.completedFuture(false); ++ } ++- Stopwatch stopwatch = Stopwatch.createStarted(); +++ ++ if (!needCommit()) { ++ return CompletableFuture.completedFuture(true); ++ } ++- try { ++- int permits = commitLock.drainPermits(); ++- if (permits <= 0) { ++- return CompletableFuture.completedFuture(false); ++- } ++- } catch (Exception e) { +++ +++ if (commitLock.drainPermits() <= 0) { ++ return CompletableFuture.completedFuture(false); ++ } ++- List bufferList = rollingUploadBuffer(); ++- int bufferSize = 0; ++- for (ByteBuffer buffer : bufferList) { ++- bufferSize += buffer.remaining(); ++- } ++- if (codaBuffer != null) { ++- bufferSize += codaBuffer.remaining(); ++- } ++- if (bufferSize == 0) { ++- return CompletableFuture.completedFuture(true); ++- } ++- TieredFileSegmentInputStream inputStream = TieredFileSegmentInputStreamFactory.build( ++- fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize); ++- int finalBufferSize = bufferSize; +++ ++ try { ++- flightCommitRequest = commit0(inputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX) +++ if (fileSegmentInputStream != null) { +++ long fileSize = this.getSize(); +++ if (fileSize == -1L) { +++ logger.error("Get commit position error before commit, Commit: %d, Expect: %d, Current Max: %d, FileName: %s", +++ commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath()); +++ releaseCommitLock(); +++ return CompletableFuture.completedFuture(false); +++ } else { +++ if (correctPosition(fileSize, null)) { +++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); +++ fileSegmentInputStream = null; +++ } +++ } +++ } +++ +++ int bufferSize; +++ if (fileSegmentInputStream != null) { +++ bufferSize = fileSegmentInputStream.available(); +++ } else { +++ List bufferList = borrowBuffer(); +++ bufferSize = bufferList.stream().mapToInt(ByteBuffer::remaining).sum() +++ + (codaBuffer != null ? codaBuffer.remaining() : 0); +++ if (bufferSize == 0) { +++ releaseCommitLock(); +++ return CompletableFuture.completedFuture(true); +++ } +++ fileSegmentInputStream = FileSegmentInputStreamFactory.build( +++ fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize); +++ } +++ +++ return flightCommitRequest = this +++ .commit0(fileSegmentInputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX) ++ .thenApply(result -> { ++ if (result) { ++- if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) { ++- dispatchCommitOffset = MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1)); ++- } ++- commitPosition += finalBufferSize; +++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); +++ commitPosition += bufferSize; +++ fileSegmentInputStream = null; ++ return true; ++- } ++- sendBackBuffer(inputStream); ++- return false; ++- }) ++- .exceptionally(e -> handleCommitException(inputStream, e)) ++- .whenComplete((result, e) -> { ++- if (commitLock.availablePermits() == 0) { ++- logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize); ++- commitLock.release(); ++ } else { ++- logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits()); +++ fileSegmentInputStream.rewind(); +++ return false; ++ } ++- }); ++- return flightCommitRequest; +++ }) +++ .exceptionally(this::handleCommitException) +++ .whenComplete((result, e) -> releaseCommitLock()); +++ ++ } catch (Exception e) { ++- handleCommitException(inputStream, e); ++- if (commitLock.availablePermits() == 0) { ++- logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize); ++- commitLock.release(); ++- } else { ++- logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits()); ++- } +++ handleCommitException(e); +++ releaseCommitLock(); ++ } ++ return CompletableFuture.completedFuture(false); ++ } ++ ++- private boolean handleCommitException(TieredFileSegmentInputStream inputStream, Throwable e) { +++ private long getCorrectFileSize(Throwable throwable) { +++ if (throwable instanceof TieredStoreException) { +++ long fileSize = ((TieredStoreException) throwable).getPosition(); +++ if (fileSize > 0) { +++ return fileSize; +++ } +++ } +++ return getSize(); +++ } +++ +++ private boolean handleCommitException(Throwable e) { +++ // Get root cause here ++ Throwable cause = e.getCause() != null ? e.getCause() : e; ++- sendBackBuffer(inputStream); ++- long realSize = 0; ++- if (cause instanceof TieredStoreException && ((TieredStoreException) cause).getPosition() > 0) { ++- realSize = ((TieredStoreException) cause).getPosition(); +++ long fileSize = this.getCorrectFileSize(cause); +++ +++ if (fileSize == -1L) { +++ logger.error("Get commit position error, Commit: %d, Expect: %d, Current Max: %d, FileName: %s", +++ commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath()); +++ fileSegmentInputStream.rewind(); +++ return false; +++ } +++ +++ if (correctPosition(fileSize, cause)) { +++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); +++ fileSegmentInputStream = null; +++ return true; +++ } else { +++ fileSegmentInputStream.rewind(); +++ return false; ++ } ++- if (realSize <= 0) { ++- realSize = getSize(); +++ } +++ +++ /** +++ * return true to clear buffer +++ */ +++ private boolean correctPosition(long fileSize, Throwable throwable) { +++ +++ // Current we have three offsets here: commit offset, expect offset, file size. +++ // We guarantee that the commit offset is less than or equal to the expect offset. +++ // Max offset will increase because we can continuously put in new buffers +++ String handleInfo = throwable == null ? "before commit" : "after commit"; +++ long expectPosition = commitPosition + fileSegmentInputStream.getContentLength(); +++ +++ String offsetInfo = String.format("Correct Commit Position, %s, result=[{}], " + +++ "Commit: %d, Expect: %d, Current Max: %d, FileSize: %d, FileName: %s", +++ handleInfo, commitPosition, expectPosition, appendPosition, fileSize, this.getPath()); +++ +++ // We are believing that the file size returned by the server is correct, +++ // can reset the commit offset to the file size reported by the storage system. +++ if (fileSize == expectPosition) { +++ logger.info(offsetInfo, "Success", throwable); +++ commitPosition = fileSize; +++ return true; ++ } ++- if (realSize > 0 && realSize > commitPosition) { ++- logger.error("TieredFileSegment#handleCommitException: commit failed: file: {}, try to fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause); ++- // TODO check if this diff part is uploaded to backend storage ++- long diff = appendPosition - commitPosition; ++- commitPosition = realSize; ++- appendPosition = realSize + diff; ++- // TODO check if appendPosition is large than maxOffset ++- } else if (realSize < commitPosition) { ++- logger.error("[Bug]TieredFileSegment#handleCommitException: commit failed: file: {}, can not fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause); +++ +++ if (fileSize < commitPosition) { +++ logger.error(offsetInfo, "FileSizeIncorrect", throwable); +++ } else if (fileSize == commitPosition) { +++ logger.warn(offsetInfo, "CommitFailed", throwable); +++ } else if (fileSize > commitPosition) { +++ logger.warn(offsetInfo, "PartialSuccess", throwable); ++ } +++ commitPosition = fileSize; ++ return false; ++ } ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java ++index 5a0ca25f5..0db3eaf8f 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java ++@@ -18,7 +18,7 @@ package org.apache.rocketmq.tieredstore.provider; ++ ++ import java.nio.ByteBuffer; ++ import java.util.concurrent.CompletableFuture; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ ++ public interface TieredStoreProvider { ++ ++@@ -30,7 +30,9 @@ public interface TieredStoreProvider { ++ String getPath(); ++ ++ /** ++- * Get file size in backend file system +++ * Get the real length of the file. +++ * Return 0 if the file does not exist, +++ * Return -1 if system get size failed. ++ * ++ * @return file real size ++ */ ++@@ -71,5 +73,5 @@ public interface TieredStoreProvider { ++ * @param append try to append or create a new file ++ * @return put result, true if data successfully write; false otherwise ++ */ ++- CompletableFuture commit0(TieredFileSegmentInputStream inputStream,long position, int length, boolean append); +++ CompletableFuture commit0(FileSegmentInputStream inputStream,long position, int length, boolean append); ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java ++index 52be90b1d..7e949cb28 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java ++@@ -36,7 +36,7 @@ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; ++ import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor; ++ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; ++ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ ++ import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_FILE_TYPE; ++@@ -184,7 +184,7 @@ public class PosixFileSegment extends TieredFileSegment { ++ ++ @Override ++ public CompletableFuture commit0( ++- TieredFileSegmentInputStream inputStream, long position, int length, boolean append) { +++ FileSegmentInputStream inputStream, long position, int length, boolean append) { ++ ++ Stopwatch stopwatch = Stopwatch.createStarted(); ++ AttributesBuilder attributesBuilder = newAttributesBuilder() ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java ++similarity index 88% ++rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java ++rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java ++index c70bb7656..13b6e0ef9 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java ++@@ -15,7 +15,7 @@ ++ * limitations under the License. ++ */ ++ ++-package org.apache.rocketmq.tieredstore.provider.inputstream; +++package org.apache.rocketmq.tieredstore.provider.stream; ++ ++ import java.io.IOException; ++ import java.nio.ByteBuffer; ++@@ -23,20 +23,23 @@ import java.util.List; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ ++-public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { +++public class CommitLogInputStream extends FileSegmentInputStream { ++ ++ /** ++ * commitLogOffset is the real physical offset of the commitLog buffer which is being read ++ */ +++ private final long startCommitLogOffset; +++ ++ private long commitLogOffset; ++ ++ private final ByteBuffer codaBuffer; ++ ++ private long markCommitLogOffset = -1; ++ ++- public TieredCommitLogInputStream(FileSegmentType fileType, long startOffset, +++ public CommitLogInputStream(FileSegmentType fileType, long startOffset, ++ List uploadBufferList, ByteBuffer codaBuffer, int contentLength) { ++ super(fileType, uploadBufferList, contentLength); +++ this.startCommitLogOffset = startOffset; ++ this.commitLogOffset = startOffset; ++ this.codaBuffer = codaBuffer; ++ } ++@@ -53,6 +56,15 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { ++ this.commitLogOffset = markCommitLogOffset; ++ } ++ +++ @Override +++ public synchronized void rewind() { +++ super.rewind(); +++ this.commitLogOffset = this.startCommitLogOffset; +++ if (this.codaBuffer != null) { +++ this.codaBuffer.rewind(); +++ } +++ } +++ ++ @Override ++ public ByteBuffer getCodaBuffer() { ++ return this.codaBuffer; ++@@ -64,17 +76,17 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { ++ return -1; ++ } ++ readPosition++; ++- if (curReadBufferIndex >= uploadBufferList.size()) { +++ if (curReadBufferIndex >= bufferList.size()) { ++ return readCoda(); ++ } ++ int res; ++ if (readPosInCurBuffer >= curBuffer.remaining()) { ++ curReadBufferIndex++; ++- if (curReadBufferIndex >= uploadBufferList.size()) { +++ if (curReadBufferIndex >= bufferList.size()) { ++ readPosInCurBuffer = 0; ++ return readCoda(); ++ } ++- curBuffer = uploadBufferList.get(curReadBufferIndex); +++ curBuffer = bufferList.get(curReadBufferIndex); ++ commitLogOffset += readPosInCurBuffer; ++ readPosInCurBuffer = 0; ++ } ++@@ -119,9 +131,9 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { ++ int posInCurBuffer = readPosInCurBuffer; ++ long curCommitLogOffset = commitLogOffset; ++ ByteBuffer curBuf = curBuffer; ++- while (needRead > 0 && bufIndex <= uploadBufferList.size()) { +++ while (needRead > 0 && bufIndex <= bufferList.size()) { ++ int readLen, remaining, realReadLen = 0; ++- if (bufIndex == uploadBufferList.size()) { +++ if (bufIndex == bufferList.size()) { ++ // read from coda buffer ++ remaining = codaBuffer.remaining() - posInCurBuffer; ++ readLen = Math.min(remaining, needRead); ++@@ -137,7 +149,7 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { ++ } ++ remaining = curBuf.remaining() - posInCurBuffer; ++ readLen = Math.min(remaining, needRead); ++- curBuf = uploadBufferList.get(bufIndex); +++ curBuf = bufferList.get(bufIndex); ++ if (posInCurBuffer < MessageBufferUtil.PHYSICAL_OFFSET_POSITION) { ++ realReadLen = Math.min(MessageBufferUtil.PHYSICAL_OFFSET_POSITION - posInCurBuffer, readLen); ++ // read from commitLog buffer ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java ++similarity index 77% ++rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java ++rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java ++index e1758ca93..9e9d5135c 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java ++@@ -15,15 +15,16 @@ ++ * limitations under the License. ++ */ ++ ++-package org.apache.rocketmq.tieredstore.provider.inputstream; +++package org.apache.rocketmq.tieredstore.provider.stream; ++ ++ import java.io.IOException; ++ import java.io.InputStream; ++ import java.nio.ByteBuffer; ++ import java.util.List; +++import org.apache.commons.collections.CollectionUtils; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ ++-public class TieredFileSegmentInputStream extends InputStream { +++public class FileSegmentInputStream extends InputStream { ++ ++ /** ++ * file type, can be commitlog, consume queue or indexfile now ++@@ -33,7 +34,7 @@ public class TieredFileSegmentInputStream extends InputStream { ++ /** ++ * hold bytebuffer ++ */ ++- protected final List uploadBufferList; +++ protected final List bufferList; ++ ++ /** ++ * total remaining of bytebuffer list ++@@ -65,13 +66,13 @@ public class TieredFileSegmentInputStream extends InputStream { ++ ++ private int markReadPosInCurBuffer = -1; ++ ++- public TieredFileSegmentInputStream(FileSegmentType fileType, List uploadBufferList, ++- int contentLength) { +++ public FileSegmentInputStream( +++ FileSegmentType fileType, List bufferList, int contentLength) { ++ this.fileType = fileType; ++ this.contentLength = contentLength; ++- this.uploadBufferList = uploadBufferList; ++- if (uploadBufferList != null && uploadBufferList.size() > 0) { ++- this.curBuffer = uploadBufferList.get(curReadBufferIndex); +++ this.bufferList = bufferList; +++ if (bufferList != null && bufferList.size() > 0) { +++ this.curBuffer = bufferList.get(curReadBufferIndex); ++ } ++ } ++ ++@@ -95,18 +96,34 @@ public class TieredFileSegmentInputStream extends InputStream { ++ this.readPosition = markReadPosition; ++ this.curReadBufferIndex = markCurReadBufferIndex; ++ this.readPosInCurBuffer = markReadPosInCurBuffer; ++- if (this.curReadBufferIndex < uploadBufferList.size()) { ++- this.curBuffer = uploadBufferList.get(curReadBufferIndex); +++ if (this.curReadBufferIndex < bufferList.size()) { +++ this.curBuffer = bufferList.get(curReadBufferIndex); ++ } ++ } ++ +++ public synchronized void rewind() { +++ this.readPosition = 0; +++ this.curReadBufferIndex = 0; +++ this.readPosInCurBuffer = 0; +++ if (CollectionUtils.isNotEmpty(bufferList)) { +++ this.curBuffer = bufferList.get(0); +++ for (ByteBuffer buffer : bufferList) { +++ buffer.rewind(); +++ } +++ } +++ } +++ +++ public int getContentLength() { +++ return contentLength; +++ } +++ ++ @Override ++ public int available() { ++ return contentLength - readPosition; ++ } ++ ++- public List getUploadBufferList() { ++- return uploadBufferList; +++ public List getBufferList() { +++ return bufferList; ++ } ++ ++ public ByteBuffer getCodaBuffer() { ++@@ -121,10 +138,10 @@ public class TieredFileSegmentInputStream extends InputStream { ++ readPosition++; ++ if (readPosInCurBuffer >= curBuffer.remaining()) { ++ curReadBufferIndex++; ++- if (curReadBufferIndex >= uploadBufferList.size()) { +++ if (curReadBufferIndex >= bufferList.size()) { ++ return -1; ++ } ++- curBuffer = uploadBufferList.get(curReadBufferIndex); +++ curBuffer = bufferList.get(curReadBufferIndex); ++ readPosInCurBuffer = 0; ++ } ++ return curBuffer.get(readPosInCurBuffer++) & 0xff; ++@@ -153,8 +170,8 @@ public class TieredFileSegmentInputStream extends InputStream { ++ int bufIndex = curReadBufferIndex; ++ int posInCurBuffer = readPosInCurBuffer; ++ ByteBuffer curBuf = curBuffer; ++- while (needRead > 0 && bufIndex < uploadBufferList.size()) { ++- curBuf = uploadBufferList.get(bufIndex); +++ while (needRead > 0 && bufIndex < bufferList.size()) { +++ curBuf = bufferList.get(bufIndex); ++ int remaining = curBuf.remaining() - posInCurBuffer; ++ int readLen = Math.min(remaining, needRead); ++ // read from curBuf ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java ++similarity index 54% ++rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java ++rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java ++index d0c983fd4..a90baff3a 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java ++@@ -15,30 +15,34 @@ ++ * limitations under the License. ++ */ ++ ++-package org.apache.rocketmq.tieredstore.provider.inputstream; +++package org.apache.rocketmq.tieredstore.provider.stream; ++ ++ import java.nio.ByteBuffer; ++ import java.util.List; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ ++-public class TieredFileSegmentInputStreamFactory { +++public class FileSegmentInputStreamFactory { ++ ++- public static TieredFileSegmentInputStream build(FileSegmentType fileType, ++- long startOffset, List uploadBufferList, ByteBuffer codaBuffer, int contentLength) { +++ public static FileSegmentInputStream build( +++ FileSegmentType fileType, long offset, List bufferList, ByteBuffer byteBuffer, int length) { +++ +++ if (bufferList == null) { +++ throw new IllegalArgumentException("bufferList is null"); +++ } ++ ++ switch (fileType) { ++ case COMMIT_LOG: ++- return new TieredCommitLogInputStream( ++- fileType, startOffset, uploadBufferList, codaBuffer, contentLength); +++ return new CommitLogInputStream( +++ fileType, offset, bufferList, byteBuffer, length); ++ case CONSUME_QUEUE: ++- return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength); +++ return new FileSegmentInputStream(fileType, bufferList, length); ++ case INDEX: ++- if (uploadBufferList.size() != 1) { ++- throw new IllegalArgumentException("uploadBufferList size in INDEX type input stream must be 1"); +++ if (bufferList.size() != 1) { +++ throw new IllegalArgumentException("buffer block size must be 1 when file type is IndexFile"); ++ } ++- return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength); +++ return new FileSegmentInputStream(fileType, bufferList, length); ++ default: ++- throw new IllegalArgumentException("fileType is not supported"); +++ throw new IllegalArgumentException("file type is not supported"); ++ } ++ } ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++index 8601392e7..2451199c2 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++@@ -130,36 +130,36 @@ public class TieredMessageStoreTest { ++ // TieredStorageLevel.DISABLE ++ properties.setProperty("tieredStorageLevel", "0"); ++ configuration.update(properties); ++- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ // TieredStorageLevel.NOT_IN_DISK ++ properties.setProperty("tieredStorageLevel", "1"); ++ configuration.update(properties); ++ when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false); ++- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); ++- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ // TieredStorageLevel.NOT_IN_MEM ++ properties.setProperty("tieredStorageLevel", "2"); ++ configuration.update(properties); ++ Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false); ++ Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true); ++- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); ++ Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(false); ++- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); ++ Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true); ++- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ // TieredStorageLevel.FORCE ++ properties.setProperty("tieredStorageLevel", "3"); ++ configuration.update(properties); ++- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ } ++ ++ @Test ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++index cc39cfbfc..7a4d05969 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++@@ -24,6 +24,7 @@ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; +++import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor; ++ import org.apache.rocketmq.tieredstore.metadata.FileSegmentMetadata; ++ import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore; ++ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; ++@@ -55,6 +56,7 @@ public class TieredFlatFileTest { ++ public void tearDown() throws IOException { ++ TieredStoreTestUtil.destroyMetadataStore(); ++ TieredStoreTestUtil.destroyTempDir(storePath); +++ TieredStoreExecutor.shutdown(); ++ } ++ ++ private List getSegmentMetadataList(TieredMetadataStore metadataStore) { ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++index 262d6645b..2da72bc7a 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++@@ -87,5 +87,7 @@ public class TieredIndexFileTest { ++ ++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++ Assert.assertEquals(1, indexList.size()); +++ +++ indexFile.destroy(); ++ } ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java ++similarity index 82% ++rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java ++rename to tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java ++index a6566b7de..3bbe41dd4 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java ++@@ -20,13 +20,13 @@ package org.apache.rocketmq.tieredstore.provider; ++ import java.io.InputStream; ++ import java.nio.ByteBuffer; ++ import java.util.List; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ ++-public class MockTieredFileSegmentInputStream extends TieredFileSegmentInputStream { +++public class MockFileSegmentInputStream extends FileSegmentInputStream { ++ ++ private final InputStream inputStream; ++ ++- public MockTieredFileSegmentInputStream(InputStream inputStream) { +++ public MockFileSegmentInputStream(InputStream inputStream) { ++ super(null, null, Integer.MAX_VALUE); ++ this.inputStream = inputStream; ++ } ++@@ -43,7 +43,7 @@ public class MockTieredFileSegmentInputStream extends TieredFileSegmentInputStre ++ } ++ ++ @Override ++- public List getUploadBufferList() { +++ public List getBufferList() { ++ return null; ++ } ++ ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java ++index a2554ba3d..743d9182c 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java ++@@ -28,8 +28,8 @@ import java.util.Random; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.file.TieredCommitLog; ++ import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest; ++ import org.junit.Assert; ++@@ -57,7 +57,7 @@ public class TieredFileSegmentInputStreamTest { ++ bufferSize += byteBuffer.remaining(); ++ } ++ ++- // build expected byte buffer for verifying the TieredFileSegmentInputStream +++ // build expected byte buffer for verifying the FileSegmentInputStream ++ ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); ++ for (ByteBuffer byteBuffer : uploadBufferList) { ++ expectedByteBuffer.put(byteBuffer); ++@@ -74,7 +74,7 @@ public class TieredFileSegmentInputStreamTest { ++ int[] batchReadSizeTestSet = { ++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION - 1, MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtil.PHYSICAL_OFFSET_POSITION + 1, MSG_LEN - 1, MSG_LEN, MSG_LEN + 1 ++ }; ++- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( +++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( ++ FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), finalBufferSize, batchReadSizeTestSet); ++ ++ } ++@@ -98,7 +98,7 @@ public class TieredFileSegmentInputStreamTest { ++ int codaBufferSize = codaBuffer.remaining(); ++ bufferSize += codaBufferSize; ++ ++- // build expected byte buffer for verifying the TieredFileSegmentInputStream +++ // build expected byte buffer for verifying the FileSegmentInputStream ++ ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); ++ for (ByteBuffer byteBuffer : uploadBufferList) { ++ expectedByteBuffer.put(byteBuffer); ++@@ -119,7 +119,7 @@ public class TieredFileSegmentInputStreamTest { ++ MSG_LEN - 1, MSG_LEN, MSG_LEN + 1, ++ bufferSize - 1, bufferSize, bufferSize + 1 ++ }; ++- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( +++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( ++ FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, codaBuffer, finalBufferSize), finalBufferSize, batchReadSizeTestSet); ++ ++ } ++@@ -134,7 +134,7 @@ public class TieredFileSegmentInputStreamTest { ++ bufferSize += byteBuffer.remaining(); ++ } ++ ++- // build expected byte buffer for verifying the TieredFileSegmentInputStream +++ // build expected byte buffer for verifying the FileSegmentInputStream ++ ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); ++ for (ByteBuffer byteBuffer : uploadBufferList) { ++ expectedByteBuffer.put(byteBuffer); ++@@ -143,7 +143,7 @@ public class TieredFileSegmentInputStreamTest { ++ ++ int finalBufferSize = bufferSize; ++ int[] batchReadSizeTestSet = {TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE - 1, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE + 1}; ++- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( +++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( ++ FileSegmentType.CONSUME_QUEUE, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), bufferSize, batchReadSizeTestSet); ++ } ++ ++@@ -156,16 +156,16 @@ public class TieredFileSegmentInputStreamTest { ++ byteBuffer.flip(); ++ List uploadBufferList = Arrays.asList(byteBuffer); ++ ++- // build expected byte buffer for verifying the TieredFileSegmentInputStream +++ // build expected byte buffer for verifying the FileSegmentInputStream ++ ByteBuffer expectedByteBuffer = byteBuffer.slice(); ++ ++- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( +++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( ++ FileSegmentType.INDEX, COMMIT_LOG_START_OFFSET, uploadBufferList, null, byteBuffer.limit()), byteBuffer.limit(), new int[] {23, 24, 25}); ++ } ++ ++- private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier constructor, +++ private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier constructor, ++ int bufferSize, int[] readBatchSizeTestSet) { ++- TieredFileSegmentInputStream inputStream = constructor.get(); +++ FileSegmentInputStream inputStream = constructor.get(); ++ ++ // verify ++ verifyInputStream(inputStream, expectedByteBuffer); ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java ++index 4cd83e0d2..a655710a5 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java ++@@ -116,13 +116,22 @@ public class TieredFileSegmentTest { ++ } ++ ++ @Test ++- public void testCommitFailed() { +++ public void testCommitFailedThenSuccess() { ++ long startTime = System.currentTimeMillis(); ++ MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG); ++ long lastSize = segment.getSize(); ++- segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0); ++- segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0); ++- +++ segment.setCheckSize(false); +++ segment.initPosition(lastSize); +++ segment.setSize((int) lastSize); +++ +++ ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( +++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize); +++ ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( +++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN); +++ segment.append(buffer1, 0); +++ segment.append(buffer2, 0); +++ +++ // Mock new message arrive ++ segment.blocker = new CompletableFuture<>(); ++ new Thread(() -> { ++ try { ++@@ -131,20 +140,88 @@ public class TieredFileSegmentTest { ++ Assert.fail(e.getMessage()); ++ } ++ ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer(); +++ buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2); ++ buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime); ++ segment.append(buffer, 0); ++ segment.blocker.complete(false); ++ }).start(); ++ +++ // Commit failed ++ segment.commit(); ++ segment.blocker.join(); +++ segment.blocker = null; +++ +++ // Copy data and assume commit success +++ segment.getMemStore().put(buffer1); +++ segment.getMemStore().put(buffer2); +++ segment.setSize((int) (lastSize + MessageBufferUtilTest.MSG_LEN * 2)); ++ ++- segment.blocker = new CompletableFuture<>(); ++- segment.blocker.complete(true); ++ segment.commit(); +++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); +++ +++ ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN); +++ Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1)); +++ +++ ByteBuffer msg2 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtil.getCommitLogOffset(msg2)); +++ +++ ByteBuffer msg3 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtilTest.MSG_LEN); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtil.getCommitLogOffset(msg3)); +++ } +++ +++ @Test +++ public void testCommitFailed3Times() { +++ long startTime = System.currentTimeMillis(); +++ MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG); +++ long lastSize = segment.getSize(); +++ segment.setCheckSize(false); +++ segment.initPosition(lastSize); +++ segment.setSize((int) lastSize); +++ +++ ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( +++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize); +++ ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( +++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN); +++ segment.append(buffer1, 0); +++ segment.append(buffer2, 0); +++ +++ // Mock new message arrive +++ segment.blocker = new CompletableFuture<>(); +++ new Thread(() -> { +++ try { +++ Thread.sleep(3000); +++ } catch (InterruptedException e) { +++ Assert.fail(e.getMessage()); +++ } +++ ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer(); +++ buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2); +++ buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime); +++ segment.append(buffer, 0); +++ segment.blocker.complete(false); +++ }).start(); +++ +++ for (int i = 0; i < 3; i++) { +++ segment.commit(); +++ } +++ +++ Assert.assertEquals(lastSize, segment.getCommitPosition()); +++ Assert.assertEquals(baseOffset + lastSize, segment.getCommitOffset()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); +++ +++ segment.blocker.join(); +++ segment.blocker = null; ++ +++ segment.commit(); +++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitPosition()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitOffset()); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); +++ +++ segment.commit(); +++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition()); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); ++ ++ ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN); ++ Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1)); ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java ++index cb155cf8f..80ad41f68 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java ++@@ -23,7 +23,7 @@ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; ++ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ import org.junit.Assert; ++ ++@@ -33,6 +33,8 @@ public class MemoryFileSegment extends TieredFileSegment { ++ ++ public CompletableFuture blocker; ++ +++ protected int size = 0; +++ ++ protected boolean checkSize = true; ++ ++ public MemoryFileSegment(FileSegmentType fileType, MessageQueue messageQueue, long baseOffset, ++@@ -56,6 +58,18 @@ public class MemoryFileSegment extends TieredFileSegment { ++ memStore.position((int) getSize()); ++ } ++ +++ public boolean isCheckSize() { +++ return checkSize; +++ } +++ +++ public void setCheckSize(boolean checkSize) { +++ this.checkSize = checkSize; +++ } +++ +++ public ByteBuffer getMemStore() { +++ return memStore; +++ } +++ ++ @Override ++ public String getPath() { ++ return filePath; ++@@ -66,7 +80,11 @@ public class MemoryFileSegment extends TieredFileSegment { ++ if (checkSize) { ++ return 1000; ++ } ++- return 0; +++ return size; +++ } +++ +++ public void setSize(int size) { +++ this.size = size; ++ } ++ ++ @Override ++@@ -85,11 +103,11 @@ public class MemoryFileSegment extends TieredFileSegment { ++ ++ @Override ++ public CompletableFuture commit0( ++- TieredFileSegmentInputStream inputStream, long position, int length, boolean append) { +++ FileSegmentInputStream inputStream, long position, int length, boolean append) { ++ ++ try { ++ if (blocker != null && !blocker.get()) { ++- throw new IllegalStateException(); +++ throw new IllegalStateException("Commit Exception for Memory Test"); ++ } ++ } catch (InterruptedException | ExecutionException e) { ++ Assert.fail(e.getMessage()); ++@@ -98,7 +116,6 @@ public class MemoryFileSegment extends TieredFileSegment { ++ Assert.assertTrue(!checkSize || position >= getSize()); ++ ++ byte[] buffer = new byte[1024]; ++- ++ int startPos = memStore.position(); ++ try { ++ int len; ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java ++index 8ac330b37..630fd2223 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java ++@@ -22,7 +22,7 @@ import java.util.concurrent.ExecutionException; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ import org.junit.Assert; ++ ++@@ -46,7 +46,7 @@ public class MemoryFileSegmentWithoutCheck extends MemoryFileSegment { ++ } ++ ++ @Override ++- public CompletableFuture commit0(TieredFileSegmentInputStream inputStream, long position, int length, +++ public CompletableFuture commit0(FileSegmentInputStream inputStream, long position, int length, ++ boolean append) { ++ try { ++ if (blocker != null && !blocker.get()) { ++-- ++2.32.0.windows.2 ++ ++ ++From d000ef947d7c99918ceba0fa451c1e29fd84ba07 Mon Sep 17 00:00:00 2001 ++From: yuz10 <845238369@qq.com> ++Date: Thu, 31 Aug 2023 09:41:33 +0800 ++Subject: [PATCH 3/7] [ISSUE #7283] Incorrect dledger commitlog min offset ++ after mappedFile re delete failed (#7284) ++ ++--- ++ .../apache/rocketmq/store/dledger/DLedgerCommitLog.java | 7 ++++++- ++ 1 file changed, 6 insertions(+), 1 deletion(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java ++index ec5e86d70..d5f6acdc0 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java ++@@ -162,7 +162,12 @@ public class DLedgerCommitLog extends CommitLog { ++ if (!mappedFileQueue.getMappedFiles().isEmpty()) { ++ return mappedFileQueue.getMinOffset(); ++ } ++- return dLedgerFileList.getMinOffset(); +++ for (MmapFile file : dLedgerFileList.getMappedFiles()) { +++ if (file.isAvailable()) { +++ return file.getFileFromOffset() + file.getStartPosition(); +++ } +++ } +++ return 0; ++ } ++ ++ @Override ++-- ++2.32.0.windows.2 ++ ++ ++From f82718ae3b77a16b553c03f672dc971a2d5d48fa Mon Sep 17 00:00:00 2001 ++From: cnScarb ++Date: Thu, 31 Aug 2023 15:50:10 +0800 ++Subject: [PATCH 4/7] [ISSUE #7208] fix: when deleting topic also delete its ++ pop retry topic (#7209) ++ ++--- ++ .../processor/AdminBrokerProcessor.java | 24 ++++++++++--- ++ .../processor/AdminBrokerProcessorTest.java | 36 +++++++++++++++++++ ++ 2 files changed, 55 insertions(+), 5 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++index bbddcec2d..8fbcd3c94 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++@@ -51,6 +51,7 @@ import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; ++ import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; ++ import org.apache.rocketmq.common.AclConfig; ++ import org.apache.rocketmq.common.BrokerConfig; +++import org.apache.rocketmq.common.KeyBuilder; ++ import org.apache.rocketmq.common.LockCallback; ++ import org.apache.rocketmq.common.MQVersion; ++ import org.apache.rocketmq.common.MixAll; ++@@ -542,16 +543,29 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ } ++ } ++ ++- this.brokerController.getTopicConfigManager().deleteTopicConfig(requestHeader.getTopic()); ++- this.brokerController.getTopicQueueMappingManager().delete(requestHeader.getTopic()); ++- this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(requestHeader.getTopic()); ++- this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(requestHeader.getTopic()); ++- this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(requestHeader.getTopic())); +++ final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic); +++ // delete pop retry topics first +++ for (String group : groups) { +++ final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); +++ if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { +++ deleteTopicInBroker(popRetryTopic); +++ } +++ } +++ // delete topic +++ deleteTopicInBroker(topic); ++ response.setCode(ResponseCode.SUCCESS); ++ response.setRemark(null); ++ return response; ++ } ++ +++ private void deleteTopicInBroker(String topic) { +++ this.brokerController.getTopicConfigManager().deleteTopicConfig(topic); +++ this.brokerController.getTopicQueueMappingManager().delete(topic); +++ this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(topic); +++ this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(topic); +++ this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(topic)); +++ } +++ ++ private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx, ++ RemotingCommand request) throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++index d33a217f7..9d17011b6 100644 ++--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++@@ -29,6 +29,7 @@ import java.util.HashMap; ++ import java.util.Map; ++ import java.util.Properties; ++ import java.util.Set; +++import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.atomic.LongAdder; ++ import org.apache.rocketmq.broker.BrokerController; ++@@ -41,6 +42,7 @@ import org.apache.rocketmq.broker.topic.RocksDBTopicConfigManager; ++ import org.apache.rocketmq.broker.topic.TopicConfigManager; ++ import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.BrokerConfig; +++import org.apache.rocketmq.common.KeyBuilder; ++ import org.apache.rocketmq.common.MixAll; ++ import org.apache.rocketmq.common.TopicConfig; ++ import org.apache.rocketmq.common.TopicFilterType; ++@@ -90,8 +92,11 @@ import static org.assertj.core.api.Assertions.assertThat; ++ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyInt; ++ import static org.mockito.ArgumentMatchers.anyLong; +++import static org.mockito.ArgumentMatchers.anySet; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.Mockito.mock; +++import static org.mockito.Mockito.times; +++import static org.mockito.Mockito.verify; ++ import static org.mockito.Mockito.when; ++ ++ @RunWith(MockitoJUnitRunner.class) ++@@ -321,6 +326,37 @@ public class AdminBrokerProcessorTest { ++ "please execute it from master broker."); ++ } ++ +++ @Test +++ public void testDeleteWithPopRetryTopic() throws Exception { +++ String topic = "topicA"; +++ String anotherTopic = "another_topicA"; +++ +++ topicConfigManager = mock(TopicConfigManager.class); +++ when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); +++ final ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); +++ topicConfigTable.put(topic, new TopicConfig()); +++ topicConfigTable.put(KeyBuilder.buildPopRetryTopic(topic, "cid1"), new TopicConfig()); +++ +++ topicConfigTable.put(anotherTopic, new TopicConfig()); +++ topicConfigTable.put(KeyBuilder.buildPopRetryTopic(anotherTopic, "cid2"), new TopicConfig()); +++ when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigTable); +++ when(topicConfigManager.selectTopicConfig(anyString())).thenAnswer(invocation -> { +++ final String selectTopic = invocation.getArgument(0); +++ return topicConfigManager.getTopicConfigTable().get(selectTopic); +++ }); +++ +++ when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); +++ when(consumerOffsetManager.whichGroupByTopic(topic)).thenReturn(Sets.newHashSet("cid1")); +++ +++ RemotingCommand request = buildDeleteTopicRequest(topic); +++ RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); +++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); +++ +++ verify(topicConfigManager).deleteTopicConfig(topic); +++ verify(topicConfigManager).deleteTopicConfig(KeyBuilder.buildPopRetryTopic(topic, "cid1")); +++ verify(messageStore, times(2)).deleteTopics(anySet()); +++ } +++ ++ @Test ++ public void testGetAllTopicConfigInRocksdb() throws Exception { ++ if (notToBeExecuted()) { ++-- ++2.32.0.windows.2 ++ ++ ++From 31d10385d1616445478104ce9ef463a8c4852ba2 Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Mon, 4 Sep 2023 14:09:32 +0800 ++Subject: [PATCH 5/7] [ISSUE #7289] Fixed asynchronous send backpressure ++ capability ++ ++Co-authored-by: guyinyou ++--- ++ .../impl/producer/DefaultMQProducerImpl.java | 77 +++++++++++++------ ++ 1 file changed, 53 insertions(+), 24 deletions(-) ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++index bbbb17b07..2d6b83ac2 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++@@ -547,6 +547,8 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ @Deprecated ++ public void send(final Message msg, final SendCallback sendCallback, final long timeout) ++ throws MQClientException, RemotingException, InterruptedException { +++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); +++ ++ final long beginStartTime = System.currentTimeMillis(); ++ Runnable runnable = new Runnable() { ++ @Override ++@@ -554,20 +556,53 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ long costTime = System.currentTimeMillis() - beginStartTime; ++ if (timeout > costTime) { ++ try { ++- sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime); +++ sendDefaultImpl(msg, CommunicationMode.ASYNC, newCallBack, timeout - costTime); ++ } catch (Exception e) { ++- sendCallback.onException(e); +++ newCallBack.onException(e); ++ } ++ } else { ++- sendCallback.onException( +++ newCallBack.onException( ++ new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout")); ++ } ++ } ++ }; ++- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); +++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); ++ } ++ ++- public void executeAsyncMessageSend(Runnable runnable, final Message msg, final SendCallback sendCallback, +++ class BackpressureSendCallBack implements SendCallback { +++ public boolean isSemaphoreAsyncSizeAquired = false; +++ public boolean isSemaphoreAsyncNumAquired = false; +++ public int msgLen; +++ private final SendCallback sendCallback; +++ +++ public BackpressureSendCallBack(final SendCallback sendCallback) { +++ this.sendCallback = sendCallback; +++ } +++ +++ @Override +++ public void onSuccess(SendResult sendResult) { +++ if (isSemaphoreAsyncSizeAquired) { +++ semaphoreAsyncSendSize.release(msgLen); +++ } +++ if (isSemaphoreAsyncNumAquired) { +++ semaphoreAsyncSendNum.release(); +++ } +++ sendCallback.onSuccess(sendResult); +++ } +++ +++ @Override +++ public void onException(Throwable e) { +++ if (isSemaphoreAsyncSizeAquired) { +++ semaphoreAsyncSendSize.release(msgLen); +++ } +++ if (isSemaphoreAsyncNumAquired) { +++ semaphoreAsyncSendNum.release(); +++ } +++ sendCallback.onException(e); +++ } +++ } +++ +++ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final BackpressureSendCallBack sendCallback, ++ final long timeout, final long beginStartTime) ++ throws MQClientException, InterruptedException { ++ ExecutorService executor = this.getAsyncSenderExecutor(); ++@@ -595,7 +630,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ return; ++ } ++ } ++- +++ sendCallback.isSemaphoreAsyncSizeAquired = isSemaphoreAsyncSizeAquired; +++ sendCallback.isSemaphoreAsyncNumAquired = isSemaphoreAsyncNumAquired; +++ sendCallback.msgLen = msgLen; ++ executor.submit(runnable); ++ } catch (RejectedExecutionException e) { ++ if (isEnableBackpressureForAsyncMode) { ++@@ -603,15 +640,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ } else { ++ throw new MQClientException("executor rejected ", e); ++ } ++- } finally { ++- if (isSemaphoreAsyncSizeAquired) { ++- semaphoreAsyncSendSize.release(msgLen); ++- } ++- if (isSemaphoreAsyncNumAquired) { ++- semaphoreAsyncSendNum.release(); ++- } ++ } ++- ++ } ++ ++ public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg, ++@@ -1188,7 +1217,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ @Deprecated ++ public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) ++ throws MQClientException, RemotingException, InterruptedException { ++- +++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); ++ final long beginStartTime = System.currentTimeMillis(); ++ Runnable runnable = new Runnable() { ++ @Override ++@@ -1203,22 +1232,22 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ long costTime = System.currentTimeMillis() - beginStartTime; ++ if (timeout > costTime) { ++ try { ++- sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, +++ sendKernelImpl(msg, mq, CommunicationMode.ASYNC, newCallBack, null, ++ timeout - costTime); ++ } catch (MQBrokerException e) { ++ throw new MQClientException("unknown exception", e); ++ } ++ } else { ++- sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); +++ newCallBack.onException(new RemotingTooMuchRequestException("call timeout")); ++ } ++ } catch (Exception e) { ++- sendCallback.onException(e); +++ newCallBack.onException(e); ++ } ++ } ++ ++ }; ++ ++- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); +++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); ++ } ++ ++ /** ++@@ -1315,7 +1344,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ public void send(final Message msg, final MessageQueueSelector selector, final Object arg, ++ final SendCallback sendCallback, final long timeout) ++ throws MQClientException, RemotingException, InterruptedException { ++- +++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); ++ final long beginStartTime = System.currentTimeMillis(); ++ Runnable runnable = new Runnable() { ++ @Override ++@@ -1324,21 +1353,21 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ if (timeout > costTime) { ++ try { ++ try { ++- sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, +++ sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, newCallBack, ++ timeout - costTime); ++ } catch (MQBrokerException e) { ++ throw new MQClientException("unknown exception", e); ++ } ++ } catch (Exception e) { ++- sendCallback.onException(e); +++ newCallBack.onException(e); ++ } ++ } else { ++- sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); +++ newCallBack.onException(new RemotingTooMuchRequestException("call timeout")); ++ } ++ } ++ ++ }; ++- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); +++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); ++ } ++ ++ /** ++-- ++2.32.0.windows.2 ++ ++ ++From d67b9d64cbd53824798af57ba18770e0fcefa37a Mon Sep 17 00:00:00 2001 ++From: yuz10 <845238369@qq.com> ++Date: Wed, 6 Sep 2023 14:07:23 +0800 ++Subject: [PATCH 6/7] [ISSUE #7302] Fix singleTopicRegister code deleted in ++ merge ++ ++--- ++ .../apache/rocketmq/broker/topic/TopicConfigManager.java | 6 +++++- ++ 1 file changed, 5 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++index 1c3b9711f..4e3c1736c 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++@@ -330,7 +330,11 @@ public class TopicConfigManager extends ConfigManager { ++ log.error("createTopicIfAbsent ", e); ++ } ++ if (createNew && register) { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ return getTopicConfig(topicConfig.getTopicName()); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 37017dbaec5c521fd529ef4aecf3658092884f84 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 6 Sep 2023 15:23:15 +0800 ++Subject: [PATCH 7/7] [ISSUE #7305] Fix metrics and transactional module not ++ shutdown while broker offline cause coredump(#7307) ++ ++--- ++ .../java/org/apache/rocketmq/broker/BrokerController.java | 8 ++++++++ ++ .../queue/TransactionalMessageServiceImpl.java | 4 +++- ++ 2 files changed, 11 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index e8f943702..6aba70cb2 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1302,6 +1302,10 @@ public class BrokerController { ++ this.fastRemotingServer.shutdown(); ++ } ++ +++ if (this.brokerMetricsManager != null) { +++ this.brokerMetricsManager.shutdown(); +++ } +++ ++ if (this.brokerStatsManager != null) { ++ this.brokerStatsManager.shutdown(); ++ } ++@@ -1324,6 +1328,10 @@ public class BrokerController { ++ this.ackMessageProcessor.shutdownPopReviveService(); ++ } ++ +++ if (this.transactionalMessageService != null) { +++ this.transactionalMessageService.close(); +++ } +++ ++ if (this.notificationProcessor != null) { ++ this.notificationProcessor.getPopLongPollingService().shutdown(); ++ } ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java ++index 93fa725a9..48db828e0 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java ++@@ -629,7 +629,9 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ ++ ++ @Override ++ public void close() { ++- +++ if (this.transactionalOpBatchService != null) { +++ this.transactionalOpBatchService.shutdown(); +++ } ++ } ++ ++ public Message getOpMessage(int queueId, String moreData) { ++-- ++2.32.0.windows.2 ++ +diff --git a/patch016-backport-Optimize-fault-tolerant-mechanism.patch b/patch016-backport-Optimize-fault-tolerant-mechanism.patch +new file mode 100644 +index 000000000..9d8ef20a1 +--- /dev/null ++++ b/patch016-backport-Optimize-fault-tolerant-mechanism.patch +@@ -0,0 +1,520 @@ ++From e11e29419f6e2d1d9673d0329e57b824ebf3da47 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 6 Sep 2023 20:42:24 +0800 ++Subject: [PATCH 1/3] [ISSUE #7308] Adding topic blacklist and filter in tiered ++ storage module (#7310) ++ ++--- ++ .../tieredstore/TieredDispatcher.java | 21 +++++++-- ++ .../tieredstore/TieredMessageStore.java | 1 + ++ .../file/TieredFlatFileManager.java | 17 ++++--- ++ .../TieredStoreTopicBlackListFilter.java | 45 +++++++++++++++++++ ++ .../provider/TieredStoreTopicFilter.java | 25 +++++++++++ ++ .../TieredStoreTopicBlackListFilterTest.java | 36 +++++++++++++++ ++ 6 files changed, 136 insertions(+), 9 deletions(-) ++ create mode 100644 tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++ create mode 100644 tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++ create mode 100644 tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++index 430c2b62e..766c559e9 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++@@ -48,6 +48,8 @@ import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile; ++ import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager; ++ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant; ++ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; +++import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicBlackListFilter; +++import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicFilter; ++ import org.apache.rocketmq.tieredstore.util.CQItemBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++@@ -56,6 +58,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ ++ private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); ++ +++ private TieredStoreTopicFilter topicFilter; ++ private final String brokerName; ++ private final MessageStore defaultStore; ++ private final TieredMessageStoreConfig storeConfig; ++@@ -70,15 +73,15 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ this.defaultStore = defaultStore; ++ this.storeConfig = storeConfig; ++ this.brokerName = storeConfig.getBrokerName(); +++ this.topicFilter = new TieredStoreTopicBlackListFilter(); ++ this.tieredFlatFileManager = TieredFlatFileManager.getInstance(storeConfig); ++ this.dispatchRequestReadMap = new ConcurrentHashMap<>(); ++ this.dispatchRequestWriteMap = new ConcurrentHashMap<>(); ++ this.dispatchTaskLock = new ReentrantLock(); ++ this.dispatchWriteLock = new ReentrantLock(); ++- this.initScheduleTask(); ++ } ++ ++- private void initScheduleTask() { +++ protected void initScheduleTask() { ++ TieredStoreExecutor.commonScheduledExecutor.scheduleWithFixedDelay(() -> ++ tieredFlatFileManager.deepCopyFlatFileToList().forEach(flatFile -> { ++ if (!flatFile.getCompositeFlatFileLock().isLocked()) { ++@@ -87,6 +90,14 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ }), 30, 10, TimeUnit.SECONDS); ++ } ++ +++ public TieredStoreTopicFilter getTopicFilter() { +++ return topicFilter; +++ } +++ +++ public void setTopicFilter(TieredStoreTopicFilter topicFilter) { +++ this.topicFilter = topicFilter; +++ } +++ ++ @Override ++ public void dispatch(DispatchRequest request) { ++ if (stopped) { ++@@ -94,7 +105,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ } ++ ++ String topic = request.getTopic(); ++- if (TieredStoreUtil.isSystemTopic(topic)) { +++ if (topicFilter != null && topicFilter.filterTopic(topic)) { ++ return; ++ } ++ ++@@ -219,6 +230,10 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ return; ++ } ++ +++ if (topicFilter != null && topicFilter.filterTopic(flatFile.getMessageQueue().getTopic())) { +++ return; +++ } +++ ++ if (flatFile.getDispatchOffset() == -1L) { ++ return; ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index 78e855f36..9fb1b2f01 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -90,6 +90,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ boolean loadNextStore = next.load(); ++ boolean result = loadFlatFile && loadNextStore; ++ if (result) { +++ dispatcher.initScheduleTask(); ++ dispatcher.start(); ++ } ++ return result; ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++index e9ae4a5a5..7c744af3b 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++@@ -134,21 +134,21 @@ public class TieredFlatFileManager { ++ public void doCleanExpiredFile() { ++ long expiredTimeStamp = System.currentTimeMillis() - ++ TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime()); ++- Random random = new Random(); ++ for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) { ++- int delay = random.nextInt(storeConfig.getMaxCommitJitter()); ++- TieredStoreExecutor.cleanExpiredFileExecutor.schedule(() -> { +++ TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> { ++ flatFile.getCompositeFlatFileLock().lock(); ++ try { ++ flatFile.cleanExpiredFile(expiredTimeStamp); ++ flatFile.destroyExpiredFile(); ++ if (flatFile.getConsumeQueueBaseOffset() == -1) { +++ logger.info("Clean flatFile because file not initialized, topic={}, queueId={}", +++ flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId()); ++ destroyCompositeFile(flatFile.getMessageQueue()); ++ } ++ } finally { ++ flatFile.getCompositeFlatFileLock().unlock(); ++ } ++- }, delay, TimeUnit.MILLISECONDS); +++ }); ++ } ++ if (indexFile != null) { ++ indexFile.cleanExpiredFile(expiredTimeStamp); ++@@ -218,8 +218,13 @@ public class TieredFlatFileManager { ++ storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId())); ++ queueCount.incrementAndGet(); ++ }); ++- logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", ++- topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); +++ +++ if (queueCount.get() == 0L) { +++ metadataStore.deleteTopic(topicMetadata.getTopic()); +++ } else { +++ logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", +++ topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); +++ } ++ } catch (Exception e) { ++ logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e); ++ } finally { ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++new file mode 100644 ++index 000000000..50adbb713 ++--- /dev/null +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++@@ -0,0 +1,45 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.tieredstore.provider; +++ +++import java.util.HashSet; +++import java.util.Set; +++import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; +++ +++public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter { +++ +++ private final Set topicBlackSet; +++ +++ public TieredStoreTopicBlackListFilter() { +++ this.topicBlackSet = new HashSet<>(); +++ } +++ +++ @Override +++ public boolean filterTopic(String topicName) { +++ if (StringUtils.isBlank(topicName)) { +++ return true; +++ } +++ return TieredStoreUtil.isSystemTopic(topicName) || topicBlackSet.contains(topicName); +++ } +++ +++ @Override +++ public void addTopicToWhiteList(String topicName) { +++ this.topicBlackSet.add(topicName); +++ } +++} ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++new file mode 100644 ++index 000000000..3f26b8b02 ++--- /dev/null +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++@@ -0,0 +1,25 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.tieredstore.provider; +++ +++public interface TieredStoreTopicFilter { +++ +++ boolean filterTopic(String topicName); +++ +++ void addTopicToWhiteList(String topicName); +++} ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++new file mode 100644 ++index 000000000..2bf48173c ++--- /dev/null +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++@@ -0,0 +1,36 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tieredstore.provider; +++ +++import org.apache.rocketmq.common.topic.TopicValidator; +++import org.junit.Assert; +++import org.junit.Test; +++ +++public class TieredStoreTopicBlackListFilterTest { +++ +++ @Test +++ public void filterTopicTest() { +++ TieredStoreTopicFilter topicFilter = new TieredStoreTopicBlackListFilter(); +++ Assert.assertTrue(topicFilter.filterTopic("")); +++ Assert.assertTrue(topicFilter.filterTopic(TopicValidator.SYSTEM_TOPIC_PREFIX + "_Topic")); +++ +++ String topicName = "WhiteTopic"; +++ Assert.assertFalse(topicFilter.filterTopic(topicName)); +++ topicFilter.addTopicToWhiteList(topicName); +++ Assert.assertTrue(topicFilter.filterTopic(topicName)); +++ } +++} ++\ No newline at end of file ++-- ++2.32.0.windows.2 ++ ++ ++From 628020537fa7035226bc8dcde9fa33d9d5df30ff Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Thu, 7 Sep 2023 16:17:47 +0800 ++Subject: [PATCH 2/3] [ISSUE #7293] Fix NPE when alter sync state set ++ ++--- ++ .../rocketmq/controller/impl/manager/ReplicasInfoManager.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java ++index b0a67531d..d83a690f9 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java ++@@ -104,7 +104,7 @@ public class ReplicasInfoManager { ++ } ++ ++ // Check master ++- if (!syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) { +++ if (syncStateInfo.getMasterBrokerId() == null || !syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) { ++ String err = String.format("Rejecting alter syncStateSet request because the current leader is:{%s}, not {%s}", ++ syncStateInfo.getMasterBrokerId(), request.getMasterBrokerId()); ++ LOGGER.error("{}", err); ++-- ++2.32.0.windows.2 ++ ++ ++From 6fd0073d6475c539e8f4c30dc4f104a56a21d724 Mon Sep 17 00:00:00 2001 ++From: Ji Juntao ++Date: Thu, 7 Sep 2023 20:21:16 +0800 ++Subject: [PATCH 3/3] [ISSUE #7319] Optimize fault-tolerant mechanism for ++ sending messages and hot update switch (#7320) ++ ++--- ++ .../impl/producer/DefaultMQProducerImpl.java | 8 ++------ ++ .../client/latency/LatencyFaultTolerance.java | 14 +++++++++++++ ++ .../latency/LatencyFaultToleranceImpl.java | 13 +++++++++++- ++ .../client/latency/MQFaultStrategy.java | 20 +++++++------------ ++ .../proxy/service/route/MessageQueueView.java | 9 --------- ++ .../service/route/TopicRouteService.java | 10 +++++++++- ++ 6 files changed, 44 insertions(+), 30 deletions(-) ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++index 2d6b83ac2..b0c212e46 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++@@ -263,9 +263,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ mQClientFactory.start(); ++ } ++ ++- if (this.mqFaultStrategy.isStartDetectorEnable()) { ++- this.mqFaultStrategy.startDetector(); ++- } +++ this.mqFaultStrategy.startDetector(); ++ ++ log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), ++ this.defaultMQProducer.isSendMessageWithVIPChannel()); ++@@ -311,9 +309,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ if (shutdownFactory) { ++ this.mQClientFactory.shutdown(); ++ } ++- if (this.mqFaultStrategy.isStartDetectorEnable()) { ++- this.mqFaultStrategy.shutdown(); ++- } +++ this.mqFaultStrategy.shutdown(); ++ RequestFutureHolder.getInstance().shutdown(this); ++ log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); ++ this.serviceState = ServiceState.SHUTDOWN_ALREADY; ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++index 72d2f3450..17aaa266a 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++@@ -89,4 +89,18 @@ public interface LatencyFaultTolerance { ++ * @param detectInterval each broker's detecting interval ++ */ ++ void setDetectInterval(final int detectInterval); +++ +++ /** +++ * Use it to set the detector work or not. +++ * +++ * @param startDetectorEnable set the detector's work status +++ */ +++ void setStartDetectorEnable(final boolean startDetectorEnable); +++ +++ /** +++ * Use it to judge if the detector enabled. +++ * +++ * @return is the detector should be started. +++ */ +++ boolean isStartDetectorEnable(); ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++index 8af629574..d3ff7eb45 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++@@ -37,6 +37,8 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ private int detectTimeout = 200; ++ private int detectInterval = 2000; ++ private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex(); +++ +++ private volatile boolean startDetectorEnable = false; ++ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { ++ @Override ++ public Thread newThread(Runnable r) { ++@@ -80,7 +82,9 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ @Override ++ public void run() { ++ try { ++- detectByOneRound(); +++ if (startDetectorEnable) { +++ detectByOneRound(); +++ } ++ } catch (Exception e) { ++ log.warn("Unexpected exception raised while detecting service reachability", e); ++ } ++@@ -137,6 +141,13 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ this.faultItemTable.remove(name); ++ } ++ +++ public boolean isStartDetectorEnable() { +++ return startDetectorEnable; +++ } +++ +++ public void setStartDetectorEnable(boolean startDetectorEnable) { +++ this.startDetectorEnable = startDetectorEnable; +++ } ++ @Override ++ public String pickOneAtLeast() { ++ final Enumeration elements = this.faultItemTable.elements(); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++index c01490784..69fb533e5 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++@@ -24,8 +24,8 @@ import org.apache.rocketmq.common.message.MessageQueue; ++ ++ public class MQFaultStrategy { ++ private LatencyFaultTolerance latencyFaultTolerance; ++- private boolean sendLatencyFaultEnable; ++- private boolean startDetectorEnable; +++ private volatile boolean sendLatencyFaultEnable; +++ private volatile boolean startDetectorEnable; ++ private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L}; ++ private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L}; ++ ++@@ -64,11 +64,11 @@ public class MQFaultStrategy { ++ ++ ++ public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) { ++- this.setStartDetectorEnable(cc.isStartDetectorEnable()); ++- this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); ++ this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector); ++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); ++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); +++ this.setStartDetectorEnable(cc.isStartDetectorEnable()); +++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); ++ } ++ ++ // For unit test. ++@@ -123,21 +123,15 @@ public class MQFaultStrategy { ++ ++ public void setStartDetectorEnable(boolean startDetectorEnable) { ++ this.startDetectorEnable = startDetectorEnable; +++ this.latencyFaultTolerance.setStartDetectorEnable(startDetectorEnable); ++ } ++ ++ public void startDetector() { ++- // user should start the detector ++- // and the thread should not be in running state. ++- if (this.sendLatencyFaultEnable && this.startDetectorEnable) { ++- // start the detector. ++- this.latencyFaultTolerance.startDetector(); ++- } +++ this.latencyFaultTolerance.startDetector(); ++ } ++ ++ public void shutdown() { ++- if (this.sendLatencyFaultEnable && this.startDetectorEnable) { ++- this.latencyFaultTolerance.shutdown(); ++- } +++ this.latencyFaultTolerance.shutdown(); ++ } ++ ++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++index 8b3c2f7c8..898e529f8 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++@@ -26,7 +26,6 @@ public class MessageQueueView { ++ private final MessageQueueSelector readSelector; ++ private final MessageQueueSelector writeSelector; ++ private final TopicRouteWrapper topicRouteWrapper; ++- private MQFaultStrategy mqFaultStrategy; ++ ++ public MessageQueueView(String topic, TopicRouteData topicRouteData, MQFaultStrategy mqFaultStrategy) { ++ this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic); ++@@ -67,12 +66,4 @@ public class MessageQueueView { ++ .add("topicRouteWrapper", topicRouteWrapper) ++ .toString(); ++ } ++- ++- public MQFaultStrategy getMQFaultStrategy() { ++- return mqFaultStrategy; ++- } ++- ++- public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { ++- this.mqFaultStrategy = mqFaultStrategy; ++- } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++index 74769a423..caf62a1e0 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++@@ -127,7 +127,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ @Override ++ public String resolve(String name) { ++ try { ++- String brokerAddr = getBrokerAddr(null, name); +++ String brokerAddr = getBrokerAddr(ProxyContext.createForInner("MQFaultStrategy"), name); ++ return brokerAddr; ++ } catch (Exception e) { ++ return null; ++@@ -175,9 +175,17 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ ++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, ++ boolean reachable) { +++ checkSendFaultToleranceEnable(); ++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); ++ } ++ +++ public void checkSendFaultToleranceEnable() { +++ boolean hotLatencySwitch = ConfigurationManager.getProxyConfig().isSendLatencyEnable(); +++ boolean hotDetectorSwitch = ConfigurationManager.getProxyConfig().isStartDetectorEnable(); +++ this.mqFaultStrategy.setSendLatencyFaultEnable(hotLatencySwitch); +++ this.mqFaultStrategy.setStartDetectorEnable(hotDetectorSwitch); +++ } +++ ++ public MQFaultStrategy getMqFaultStrategy() { ++ return this.mqFaultStrategy; ++ } ++-- ++2.32.0.windows.2 ++ +diff --git a/patch017-backport-Convergent-thread-pool-creation.patch b/patch017-backport-Convergent-thread-pool-creation.patch +new file mode 100644 +index 000000000..92d0bd2e9 +--- /dev/null ++++ b/patch017-backport-Convergent-thread-pool-creation.patch +@@ -0,0 +1,2243 @@ ++From c100d815d754d7cb330bc63e145bafd2d9b59cb1 Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Mon, 11 Sep 2023 10:13:56 +0800 ++Subject: [PATCH 1/6] [ISSUE #7328] Convergent thread pool creation (#7329) ++ ++* Convergence thread pool creation to facilitate subsequent iteration management ++ ++* Convergence thread pool creation in ThreadPoolMonitor.java ++ ++* fix unit test ++ ++* Convergence ThreadPool constructor ++ ++* Convergence ScheduledThreadPool constructor ++ ++* remove unused import ++ ++* Convergence ScheduledThreadPool constructor ++ ++* remove unused import ++ ++--------- ++--- ++ .../rocketmq/broker/BrokerController.java | 39 +++++----- ++ .../client/ClientHousekeepingService.java | 4 +- ++ .../DefaultConsumerIdsChangeListener.java | 3 +- ++ .../broker/controller/ReplicasManager.java | 9 +-- ++ .../dledger/DLedgerRoleChangeHandler.java | 4 +- ++ .../broker/failover/EscapeBridge.java | 4 +- ++ .../broker/latency/BrokerFastFailure.java | 5 +- ++ .../BrokerFixedThreadPoolExecutor.java | 57 -------------- ++ .../broker/latency/FutureTaskExt.java | 39 ---------- ++ .../rocketmq/broker/out/BrokerOuterAPI.java | 7 +- ++ .../schedule/ScheduleMessageService.java | 7 +- ++ .../broker/topic/TopicRouteInfoManager.java | 4 +- ++ ...ractTransactionalMessageCheckListener.java | 4 +- ++ .../rocketmq/broker/BrokerControllerTest.java | 2 +- ++ .../broker/latency/BrokerFastFailureTest.java | 1 + ++ .../common/config/AbstractRocksDBStorage.java | 6 +- ++ .../FutureTaskExtThreadPoolExecutor.java | 3 +- ++ .../common/thread/ThreadPoolMonitor.java | 6 +- ++ .../rocketmq/common/utils/ThreadUtils.java | 74 ++++++++++++++++--- ++ .../rocketmq/container/BrokerContainer.java | 6 +- ++ .../controller/ControllerManager.java | 14 +--- ++ .../controller/impl/DLedgerController.java | 10 +-- ++ .../DefaultBrokerHeartbeatManager.java | 3 +- ++ .../rocketmq/namesrv/NamesrvController.java | 22 ++---- ++ .../grpc/v2/channel/GrpcChannelManager.java | 6 +- ++ .../remoting/RemotingProtocolServer.java | 4 +- ++ .../proxy/service/ClusterServiceManager.java | 12 +-- ++ .../proxy/service/LocalServiceManager.java | 4 +- ++ .../receipt/DefaultReceiptHandleManager.java | 8 +- ++ .../service/route/TopicRouteService.java | 9 +-- ++ .../remoting/netty/NettyRemotingClient.java | 4 +- ++ .../remoting/netty/NettyRemotingServer.java | 4 +- ++ .../rocketmq/store/DefaultMessageStore.java | 8 +- ++ .../ha/autoswitch/AutoSwitchHAService.java | 38 +++++----- ++ .../rocketmq/store/kv/CompactionStore.java | 21 +++--- ++ .../store/queue/ConsumeQueueStore.java | 4 +- ++ .../store/stats/BrokerStatsManager.java | 14 ++-- ++ .../store/timer/TimerMessageStore.java | 6 +- ++ .../apache/rocketmq/test/util/StatUtil.java | 1 - ++ .../common/TieredStoreExecutor.java | 14 ++-- ++ .../tools/admin/DefaultMQAdminExtImpl.java | 3 +- ++ 41 files changed, 215 insertions(+), 278 deletions(-) ++ delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java ++ delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 6aba70cb2..275b64b1a 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -34,7 +34,6 @@ import org.apache.rocketmq.broker.failover.EscapeBridge; ++ import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap; ++ import org.apache.rocketmq.broker.filter.ConsumerFilterManager; ++ import org.apache.rocketmq.broker.latency.BrokerFastFailure; ++-import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; ++ import org.apache.rocketmq.broker.longpolling.LmqPullRequestHoldService; ++ import org.apache.rocketmq.broker.longpolling.NotifyMessageArrivingListener; ++ import org.apache.rocketmq.broker.longpolling.PullRequestHoldService; ++@@ -98,6 +97,7 @@ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++ import org.apache.rocketmq.common.stats.MomentStatsItem; ++ import org.apache.rocketmq.common.utils.ServiceProvider; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.Configuration; ++@@ -160,7 +160,6 @@ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ScheduledFuture; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.locks.Lock; ++ import java.util.concurrent.locks.ReentrantLock; ++@@ -455,10 +454,10 @@ public class BrokerController { ++ * Initialize resources including remoting server and thread executors. ++ */ ++ protected void initializeResources() { ++- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("BrokerControllerScheduledThread", true, getBrokerIdentity())); ++ ++- this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.sendMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getSendMessageThreadPoolNums(), ++ this.brokerConfig.getSendMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -466,7 +465,7 @@ public class BrokerController { ++ this.sendThreadPoolQueue, ++ new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity())); ++ ++- this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.pullMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getPullMessageThreadPoolNums(), ++ this.brokerConfig.getPullMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -474,7 +473,7 @@ public class BrokerController { ++ this.pullThreadPoolQueue, ++ new ThreadFactoryImpl("PullMessageThread_", getBrokerIdentity())); ++ ++- this.litePullMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.litePullMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getLitePullMessageThreadPoolNums(), ++ this.brokerConfig.getLitePullMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -482,7 +481,7 @@ public class BrokerController { ++ this.litePullThreadPoolQueue, ++ new ThreadFactoryImpl("LitePullMessageThread_", getBrokerIdentity())); ++ ++- this.putMessageFutureExecutor = new BrokerFixedThreadPoolExecutor( +++ this.putMessageFutureExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getPutMessageFutureThreadPoolNums(), ++ this.brokerConfig.getPutMessageFutureThreadPoolNums(), ++ 1000 * 60, ++@@ -490,7 +489,7 @@ public class BrokerController { ++ this.putThreadPoolQueue, ++ new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity())); ++ ++- this.ackMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.ackMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getAckMessageThreadPoolNums(), ++ this.brokerConfig.getAckMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -498,7 +497,7 @@ public class BrokerController { ++ this.ackThreadPoolQueue, ++ new ThreadFactoryImpl("AckMessageThread_", getBrokerIdentity())); ++ ++- this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.queryMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getQueryMessageThreadPoolNums(), ++ this.brokerConfig.getQueryMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -506,7 +505,7 @@ public class BrokerController { ++ this.queryThreadPoolQueue, ++ new ThreadFactoryImpl("QueryMessageThread_", getBrokerIdentity())); ++ ++- this.adminBrokerExecutor = new BrokerFixedThreadPoolExecutor( +++ this.adminBrokerExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getAdminBrokerThreadPoolNums(), ++ this.brokerConfig.getAdminBrokerThreadPoolNums(), ++ 1000 * 60, ++@@ -514,7 +513,7 @@ public class BrokerController { ++ this.adminBrokerThreadPoolQueue, ++ new ThreadFactoryImpl("AdminBrokerThread_", getBrokerIdentity())); ++ ++- this.clientManageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.clientManageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getClientManageThreadPoolNums(), ++ this.brokerConfig.getClientManageThreadPoolNums(), ++ 1000 * 60, ++@@ -522,7 +521,7 @@ public class BrokerController { ++ this.clientManagerThreadPoolQueue, ++ new ThreadFactoryImpl("ClientManageThread_", getBrokerIdentity())); ++ ++- this.heartbeatExecutor = new BrokerFixedThreadPoolExecutor( +++ this.heartbeatExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getHeartbeatThreadPoolNums(), ++ this.brokerConfig.getHeartbeatThreadPoolNums(), ++ 1000 * 60, ++@@ -530,7 +529,7 @@ public class BrokerController { ++ this.heartbeatThreadPoolQueue, ++ new ThreadFactoryImpl("HeartbeatThread_", true, getBrokerIdentity())); ++ ++- this.consumerManageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.consumerManageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getConsumerManageThreadPoolNums(), ++ this.brokerConfig.getConsumerManageThreadPoolNums(), ++ 1000 * 60, ++@@ -538,7 +537,7 @@ public class BrokerController { ++ this.consumerManagerThreadPoolQueue, ++ new ThreadFactoryImpl("ConsumerManageThread_", true, getBrokerIdentity())); ++ ++- this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.replyMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getProcessReplyMessageThreadPoolNums(), ++ this.brokerConfig.getProcessReplyMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -546,7 +545,7 @@ public class BrokerController { ++ this.replyThreadPoolQueue, ++ new ThreadFactoryImpl("ProcessReplyMessageThread_", getBrokerIdentity())); ++ ++- this.endTransactionExecutor = new BrokerFixedThreadPoolExecutor( +++ this.endTransactionExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getEndTransactionThreadPoolNums(), ++ this.brokerConfig.getEndTransactionThreadPoolNums(), ++ 1000 * 60, ++@@ -554,7 +553,7 @@ public class BrokerController { ++ this.endTransactionThreadPoolQueue, ++ new ThreadFactoryImpl("EndTransactionThread_", getBrokerIdentity())); ++ ++- this.loadBalanceExecutor = new BrokerFixedThreadPoolExecutor( +++ this.loadBalanceExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(), ++ this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(), ++ 1000 * 60, ++@@ -562,9 +561,9 @@ public class BrokerController { ++ this.loadBalanceThreadPoolQueue, ++ new ThreadFactoryImpl("LoadBalanceProcessorThread_", getBrokerIdentity())); ++ ++- this.syncBrokerMemberGroupExecutorService = new ScheduledThreadPoolExecutor(1, +++ this.syncBrokerMemberGroupExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("BrokerControllerSyncBrokerScheduledThread", getBrokerIdentity())); ++- this.brokerHeartbeatExecutorService = new ScheduledThreadPoolExecutor(1, +++ this.brokerHeartbeatExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("BrokerControllerHeartbeatScheduledThread", getBrokerIdentity())); ++ ++ this.topicQueueMappingCleanService = new TopicQueueMappingCleanService(this); ++@@ -828,8 +827,6 @@ public class BrokerController { ++ ++ initializeResources(); ++ ++- registerProcessor(); ++- ++ initializeScheduledTasks(); ++ ++ initialTransaction(); ++@@ -1690,6 +1687,8 @@ public class BrokerController { ++ } ++ } ++ }, 10, 5, TimeUnit.SECONDS); +++ +++ registerProcessor(); ++ } ++ ++ protected void scheduleSendHeartbeat() { ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java ++index 98e5f450f..cbb81f632 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java ++@@ -18,11 +18,11 @@ package org.apache.rocketmq.broker.client; ++ ++ import io.netty.channel.Channel; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.ChannelEventListener; ++@@ -35,7 +35,7 @@ public class ClientHousekeepingService implements ChannelEventListener { ++ ++ public ClientHousekeepingService(final BrokerController brokerController) { ++ this.brokerController = brokerController; ++- scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("ClientHousekeepingScheduledThread", brokerController.getBrokerIdentity())); ++ } ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java ++index 2ce036a0f..d17a2a547 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java ++@@ -22,7 +22,6 @@ import java.util.List; ++ import java.util.Map; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.common.AbstractBrokerRunnable; ++@@ -37,7 +36,7 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen ++ private final BrokerController brokerController; ++ private final int cacheSize = 8096; ++ ++- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ ThreadUtils.newGenericThreadFactory("DefaultConsumerIdsChangeListener", true)); ++ ++ private ConcurrentHashMap> consumerChannelMap = new ConcurrentHashMap<>(cacheSize); ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++index 37c82e434..a989e6e68 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++@@ -27,10 +27,8 @@ import java.util.concurrent.ArrayBlockingQueue; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ScheduledFuture; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ ++ import org.apache.commons.lang3.StringUtils; ++@@ -42,6 +40,7 @@ import org.apache.rocketmq.common.MixAll; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.EpochEntry; ++@@ -107,9 +106,9 @@ public class ReplicasManager { ++ public ReplicasManager(final BrokerController brokerController) { ++ this.brokerController = brokerController; ++ this.brokerOuterAPI = brokerController.getBrokerOuterAPI(); ++- this.scheduledService = Executors.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity())); ++- this.executorService = Executors.newFixedThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity())); ++- this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, +++ this.scheduledService = ThreadUtils.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity())); +++ this.executorService = ThreadUtils.newThreadPoolExecutor(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity())); +++ this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, ++ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("ReplicasManager_scan_thread_", brokerController.getBrokerIdentity())); ++ this.haService = (AutoSwitchHAService) brokerController.getMessageStore().getHaService(); ++ this.brokerConfig = brokerController.getBrokerConfig(); ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java ++index 75023ee1b..e6cb97640 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java ++@@ -21,12 +21,12 @@ import io.openmessaging.storage.dledger.DLedgerServer; ++ import io.openmessaging.storage.dledger.MemberState; ++ import io.openmessaging.storage.dledger.utils.DLedgerUtils; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.Future; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.store.DefaultMessageStore; ++@@ -49,7 +49,7 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange ++ this.messageStore = messageStore; ++ this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog(); ++ this.dLegerServer = dLedgerCommitLog.getdLedgerServer(); ++- this.executorService = Executors.newSingleThreadExecutor( +++ this.executorService = ThreadUtils.newSingleThreadExecutor( ++ new ThreadFactoryImpl("DLegerRoleChangeHandler_", brokerController.getBrokerIdentity())); ++ } ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java ++index 7c350fc1d..6a0817480 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java ++@@ -24,7 +24,6 @@ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.broker.BrokerController; ++@@ -43,6 +42,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++ import org.apache.rocketmq.common.message.MessageQueue; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++@@ -72,7 +72,7 @@ public class EscapeBridge { ++ public void start() throws Exception { ++ if (brokerController.getBrokerConfig().isEnableSlaveActingMaster() && brokerController.getBrokerConfig().isEnableRemoteEscape()) { ++ final BlockingQueue asyncSenderThreadPoolQueue = new LinkedBlockingQueue<>(50000); ++- this.defaultAsyncSenderExecutor = new ThreadPoolExecutor( +++ this.defaultAsyncSenderExecutor = ThreadUtils.newThreadPoolExecutor( ++ Runtime.getRuntime().availableProcessors(), ++ Runtime.getRuntime().availableProcessors(), ++ 1000 * 60, ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java ++index d3d0bc8ba..3b6e9dc67 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java ++@@ -18,13 +18,14 @@ package org.apache.rocketmq.broker.latency; ++ ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.common.AbstractBrokerRunnable; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.UtilAll; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.future.FutureTaskExt; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.netty.RequestTask; ++@@ -43,7 +44,7 @@ public class BrokerFastFailure { ++ ++ public BrokerFastFailure(final BrokerController brokerController) { ++ this.brokerController = brokerController; ++- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("BrokerFastFailureScheduledThread", true, ++ brokerController == null ? null : brokerController.getBrokerConfig())); ++ } ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java ++deleted file mode 100644 ++index d2d1143a3..000000000 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java +++++ /dev/null ++@@ -1,57 +0,0 @@ ++-/* ++- * Licensed to the Apache Software Foundation (ASF) under one or more ++- * contributor license agreements. See the NOTICE file distributed with ++- * this work for additional information regarding copyright ownership. ++- * The ASF licenses this file to You under the Apache License, Version 2.0 ++- * (the "License"); you may not use this file except in compliance with ++- * the License. You may obtain a copy of the License at ++- * ++- * http://www.apache.org/licenses/LICENSE-2.0 ++- * ++- * Unless required by applicable law or agreed to in writing, software ++- * distributed under the License is distributed on an "AS IS" BASIS, ++- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++- * See the License for the specific language governing permissions and ++- * limitations under the License. ++- */ ++- ++-package org.apache.rocketmq.broker.latency; ++- ++-import java.util.concurrent.BlockingQueue; ++-import java.util.concurrent.RejectedExecutionHandler; ++-import java.util.concurrent.RunnableFuture; ++-import java.util.concurrent.ThreadFactory; ++-import java.util.concurrent.ThreadPoolExecutor; ++-import java.util.concurrent.TimeUnit; ++- ++-public class BrokerFixedThreadPoolExecutor extends ThreadPoolExecutor { ++- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, ++- final TimeUnit unit, ++- final BlockingQueue workQueue) { ++- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); ++- } ++- ++- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, ++- final TimeUnit unit, ++- final BlockingQueue workQueue, final ThreadFactory threadFactory) { ++- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); ++- } ++- ++- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, ++- final TimeUnit unit, ++- final BlockingQueue workQueue, final RejectedExecutionHandler handler) { ++- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler); ++- } ++- ++- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, ++- final TimeUnit unit, ++- final BlockingQueue workQueue, final ThreadFactory threadFactory, ++- final RejectedExecutionHandler handler) { ++- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); ++- } ++- ++- @Override ++- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { ++- return new FutureTaskExt<>(runnable, value); ++- } ++-} ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java ++deleted file mode 100644 ++index f132efaeb..000000000 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java +++++ /dev/null ++@@ -1,39 +0,0 @@ ++-/* ++- * Licensed to the Apache Software Foundation (ASF) under one or more ++- * contributor license agreements. See the NOTICE file distributed with ++- * this work for additional information regarding copyright ownership. ++- * The ASF licenses this file to You under the Apache License, Version 2.0 ++- * (the "License"); you may not use this file except in compliance with ++- * the License. You may obtain a copy of the License at ++- * ++- * http://www.apache.org/licenses/LICENSE-2.0 ++- * ++- * Unless required by applicable law or agreed to in writing, software ++- * distributed under the License is distributed on an "AS IS" BASIS, ++- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++- * See the License for the specific language governing permissions and ++- * limitations under the License. ++- */ ++- ++-package org.apache.rocketmq.broker.latency; ++- ++-import java.util.concurrent.Callable; ++-import java.util.concurrent.FutureTask; ++- ++-public class FutureTaskExt extends FutureTask { ++- private final Runnable runnable; ++- ++- public FutureTaskExt(final Callable callable) { ++- super(callable); ++- this.runnable = null; ++- } ++- ++- public FutureTaskExt(final Runnable runnable, final V result) { ++- super(runnable, result); ++- this.runnable = runnable; ++- } ++- ++- public Runnable getRunnable() { ++- return runnable; ++- } ++-} ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++index ae81e8b11..9dfb8127d 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++@@ -27,9 +27,9 @@ import java.util.concurrent.ArrayBlockingQueue; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.CopyOnWriteArrayList; ++ import java.util.concurrent.CountDownLatch; +++import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.StringUtils; ++-import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; ++ import org.apache.rocketmq.client.consumer.PullResult; ++ import org.apache.rocketmq.client.consumer.PullStatus; ++ import org.apache.rocketmq.client.exception.MQBrokerException; ++@@ -59,6 +59,7 @@ import org.apache.rocketmq.common.namesrv.DefaultTopAddressing; ++ import org.apache.rocketmq.common.namesrv.TopAddressing; ++ import org.apache.rocketmq.common.sysflag.PullSysFlag; ++ import org.apache.rocketmq.common.topic.TopicValidator; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++@@ -144,7 +145,7 @@ public class BrokerOuterAPI { ++ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); ++ private final RemotingClient remotingClient; ++ private final TopAddressing topAddressing = new DefaultTopAddressing(MixAll.getWSAddr()); ++- private final BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, +++ private final ExecutorService brokerOuterExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, ++ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true)); ++ private final ClientMetadata clientMetadata; ++ private final RpcClient rpcClient; ++@@ -1092,7 +1093,7 @@ public class BrokerOuterAPI { ++ throw new MQBrokerException(response.getCode(), response.getRemark()); ++ } ++ ++- public BrokerFixedThreadPoolExecutor getBrokerOuterExecutor() { +++ public ExecutorService getBrokerOuterExecutor() { ++ return brokerOuterExecutor; ++ } ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++index 297b14207..0c2e6507b 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++@@ -26,7 +26,6 @@ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicBoolean; ++ import java.util.concurrent.atomic.AtomicInteger; ++@@ -91,7 +90,7 @@ public class ScheduleMessageService extends ConfigManager { ++ public ScheduleMessageService(final BrokerController brokerController) { ++ this.brokerController = brokerController; ++ this.enableAsyncDeliver = brokerController.getMessageStoreConfig().isEnableScheduleAsyncDeliver(); ++- scheduledPersistService = new ScheduledThreadPoolExecutor(1, +++ scheduledPersistService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("ScheduleMessageServicePersistThread", true, brokerController.getBrokerConfig())); ++ } ++ ++@@ -134,9 +133,9 @@ public class ScheduleMessageService extends ConfigManager { ++ public void start() { ++ if (started.compareAndSet(false, true)) { ++ this.load(); ++- this.deliverExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_")); +++ this.deliverExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_")); ++ if (this.enableAsyncDeliver) { ++- this.handleExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_")); +++ this.handleExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_")); ++ } ++ for (Map.Entry entry : this.delayLevelTable.entrySet()) { ++ Integer level = entry.getKey(); ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java ++index b35564725..11bde5f5f 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java ++@@ -23,7 +23,6 @@ import java.util.Objects; ++ import java.util.Set; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.locks.Lock; ++@@ -36,6 +35,7 @@ import org.apache.rocketmq.common.MixAll; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.message.MessageQueue; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++@@ -66,7 +66,7 @@ public class TopicRouteInfoManager { ++ } ++ ++ public void start() { ++- this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread")); +++ this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread")); ++ ++ this.scheduledExecutorService.scheduleAtFixedRate(() -> { ++ try { ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java ++index 771d84300..982355d78 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java ++@@ -19,7 +19,6 @@ package org.apache.rocketmq.broker.transaction; ++ import io.netty.channel.Channel; ++ import java.util.concurrent.ArrayBlockingQueue; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++@@ -27,6 +26,7 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.message.MessageConst; ++ import org.apache.rocketmq.common.message.MessageExt; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; ++@@ -97,7 +97,7 @@ public abstract class AbstractTransactionalMessageCheckListener { ++ ++ public synchronized void initExecutorService() { ++ if (executorService == null) { ++- executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), +++ executorService = ThreadUtils.newThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), ++ new ThreadFactoryImpl("Transaction-msg-check-thread", brokerController.getBrokerIdentity()), new CallerRunsPolicy()); ++ } ++ } ++diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java ++index 75ad961ce..6035a20ac 100644 ++--- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java +++++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java ++@@ -23,9 +23,9 @@ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.TimeUnit; ++ ++-import org.apache.rocketmq.broker.latency.FutureTaskExt; ++ import org.apache.rocketmq.common.BrokerConfig; ++ import org.apache.rocketmq.common.UtilAll; +++import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.remoting.netty.NettyClientConfig; ++ import org.apache.rocketmq.remoting.netty.NettyServerConfig; ++ import org.apache.rocketmq.remoting.netty.RequestTask; ++diff --git a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java ++index 5d0f7f9d7..31b547cf1 100644 ++--- a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java +++++ b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.latency; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.TimeUnit; +++import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.remoting.netty.RequestTask; ++ import org.junit.Test; ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++index a720a5be3..6f19a9815 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++@@ -23,7 +23,6 @@ import java.util.List; ++ import java.util.Map; ++ import java.util.concurrent.ArrayBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.Semaphore; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++@@ -33,6 +32,7 @@ import com.google.common.collect.Maps; ++ ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.rocksdb.ColumnFamilyDescriptor; ++@@ -82,8 +82,8 @@ public abstract class AbstractRocksDBStorage { ++ private volatile boolean closed; ++ ++ private final Semaphore reloadPermit = new Semaphore(1); ++- private final ScheduledExecutorService reloadScheduler = new ScheduledThreadPoolExecutor(1, new ThreadFactoryImpl("RocksDBStorageReloadService_")); ++- private final ThreadPoolExecutor manualCompactionThread = new ThreadPoolExecutor( +++ private final ScheduledExecutorService reloadScheduler = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("RocksDBStorageReloadService_")); +++ private final ThreadPoolExecutor manualCompactionThread = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor( ++ 1, 1, 1000 * 60, TimeUnit.MILLISECONDS, ++ new ArrayBlockingQueue(1), ++ new ThreadFactoryImpl("RocksDBManualCompactionService_"), ++diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java ++index 411da9221..7b68873a9 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java +++++ b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java ++@@ -29,7 +29,8 @@ public class FutureTaskExtThreadPoolExecutor extends ThreadPoolExecutor { ++ ++ public FutureTaskExtThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, ++ TimeUnit unit, ++- BlockingQueue workQueue, ThreadFactory threadFactory, +++ BlockingQueue workQueue, +++ ThreadFactory threadFactory, ++ RejectedExecutionHandler handler) { ++ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); ++ } ++diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java ++index 49d97a5d7..1bfabbffe 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java +++++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java ++@@ -22,12 +22,12 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; ++ import java.util.Collections; ++ import java.util.List; ++ import java.util.concurrent.CopyOnWriteArrayList; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.UtilAll; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++@@ -36,7 +36,7 @@ public class ThreadPoolMonitor { ++ private static Logger waterMarkLogger = LoggerFactory.getLogger(ThreadPoolMonitor.class); ++ ++ private static final List MONITOR_EXECUTOR = new CopyOnWriteArrayList<>(); ++- private static final ScheduledExecutorService MONITOR_SCHEDULED = Executors.newSingleThreadScheduledExecutor( +++ private static final ScheduledExecutorService MONITOR_SCHEDULED = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryBuilder().setNameFormat("ThreadPoolMonitor-%d").build() ++ ); ++ ++@@ -81,7 +81,7 @@ public class ThreadPoolMonitor { ++ String name, ++ int queueCapacity, ++ List threadPoolStatusMonitors) { ++- ThreadPoolExecutor executor = new FutureTaskExtThreadPoolExecutor( +++ ThreadPoolExecutor executor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor( ++ corePoolSize, ++ maximumPoolSize, ++ keepAliveTime, ++diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java ++index 4b366d4e3..1644c6360 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java +++++ b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java ++@@ -20,38 +20,94 @@ package org.apache.rocketmq.common.utils; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; +++import java.util.concurrent.LinkedBlockingQueue; +++import java.util.concurrent.RejectedExecutionHandler; ++ import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.ThreadFactory; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.thread.FutureTaskExtThreadPoolExecutor; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ public final class ThreadUtils { ++ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME); ++ ++- public static ExecutorService newThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, ++- TimeUnit unit, BlockingQueue workQueue, String processName, boolean isDaemon) { ++- return new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon)); +++ public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) { +++ return ThreadUtils.newSingleThreadExecutor(newThreadFactory(processName, isDaemon)); ++ } ++ ++- public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) { ++- return Executors.newSingleThreadExecutor(newThreadFactory(processName, isDaemon)); +++ public static ExecutorService newSingleThreadExecutor(ThreadFactory threadFactory) { +++ return ThreadUtils.newThreadPoolExecutor(1, threadFactory); +++ } +++ +++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory) { +++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, corePoolSize, +++ 0L, TimeUnit.MILLISECONDS, +++ new LinkedBlockingQueue<>(), +++ threadFactory); +++ } +++ +++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, +++ int maximumPoolSize, +++ long keepAliveTime, +++ TimeUnit unit, BlockingQueue workQueue, +++ String processName, +++ boolean isDaemon) { +++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon)); +++ } +++ +++ public static ExecutorService newThreadPoolExecutor(final int corePoolSize, +++ final int maximumPoolSize, +++ final long keepAliveTime, +++ final TimeUnit unit, +++ final BlockingQueue workQueue, +++ final ThreadFactory threadFactory) { +++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, new ThreadPoolExecutor.AbortPolicy()); +++ } +++ +++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, +++ int maximumPoolSize, +++ long keepAliveTime, +++ TimeUnit unit, +++ BlockingQueue workQueue, +++ ThreadFactory threadFactory, +++ RejectedExecutionHandler handler) { +++ return new FutureTaskExtThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); ++ } ++ ++ public static ScheduledExecutorService newSingleThreadScheduledExecutor(String processName, boolean isDaemon) { ++- return Executors.newSingleThreadScheduledExecutor(newThreadFactory(processName, isDaemon)); +++ return ThreadUtils.newScheduledThreadPool(1, processName, isDaemon); +++ } +++ +++ public static ScheduledExecutorService newSingleThreadScheduledExecutor(ThreadFactory threadFactory) { +++ return ThreadUtils.newScheduledThreadPool(1, threadFactory); +++ } +++ +++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize) { +++ return ThreadUtils.newScheduledThreadPool(corePoolSize, Executors.defaultThreadFactory()); ++ } ++ ++- public static ScheduledExecutorService newFixedThreadScheduledPool(int nThreads, String processName, +++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, String processName, ++ boolean isDaemon) { ++- return Executors.newScheduledThreadPool(nThreads, newThreadFactory(processName, isDaemon)); +++ return ThreadUtils.newScheduledThreadPool(corePoolSize, newThreadFactory(processName, isDaemon)); +++ } +++ +++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, ThreadFactory threadFactory) { +++ return ThreadUtils.newScheduledThreadPool(corePoolSize, threadFactory, new ThreadPoolExecutor.AbortPolicy()); +++ } +++ +++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, +++ ThreadFactory threadFactory, +++ RejectedExecutionHandler handler) { +++ return new ScheduledThreadPoolExecutor(corePoolSize, threadFactory, handler); ++ } ++ ++ public static ThreadFactory newThreadFactory(String processName, boolean isDaemon) { ++- return newGenericThreadFactory("Remoting-" + processName, isDaemon); +++ return newGenericThreadFactory("ThreadUtils-" + processName, isDaemon); ++ } ++ ++ public static ThreadFactory newGenericThreadFactory(String processName) { ++diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java ++index c6446f058..5b712bc30 100644 ++--- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java +++++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java ++@@ -47,14 +47,12 @@ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ ++ public class BrokerContainer implements IBrokerContainer { ++ private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); ++ ++- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new BasicThreadFactory.Builder() ++ .namingPattern("BrokerContainerScheduledThread") ++ .daemon(true) ++@@ -143,7 +141,7 @@ public class BrokerContainer implements IBrokerContainer { ++ this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.containerClientHouseKeepingService); ++ this.fastRemotingServer = this.remotingServer.newRemotingServer(this.nettyServerConfig.getListenPort() - 2); ++ ++- this.brokerContainerExecutor = new ThreadPoolExecutor( +++ this.brokerContainerExecutor = ThreadUtils.newThreadPoolExecutor( ++ 1, ++ 1, ++ 1000 * 60, ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java ++index 7c91e70da..3e6b0eba5 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java ++@@ -25,8 +25,6 @@ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; ++ import java.util.concurrent.Future; ++ import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.RunnableFuture; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ ++ import org.apache.commons.lang3.StringUtils; ++@@ -34,8 +32,8 @@ import org.apache.rocketmq.common.ControllerConfig; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++-import org.apache.rocketmq.common.future.FutureTaskExt; ++ +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy; ++ import org.apache.rocketmq.controller.impl.DLedgerController; ++ import org.apache.rocketmq.controller.impl.heartbeat.DefaultBrokerHeartbeatManager; ++@@ -93,18 +91,14 @@ public class ControllerManager { ++ ++ public boolean initialize() { ++ this.controllerRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.controllerConfig.getControllerRequestThreadPoolQueueCapacity()); ++- this.controllerRequestExecutor = new ThreadPoolExecutor( +++ this.controllerRequestExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.controllerConfig.getControllerThreadPoolNums(), ++ this.controllerConfig.getControllerThreadPoolNums(), ++ 1000 * 60, ++ TimeUnit.MILLISECONDS, ++ this.controllerRequestThreadPoolQueue, ++- new ThreadFactoryImpl("ControllerRequestExecutorThread_")) { ++- @Override ++- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { ++- return new FutureTaskExt(runnable, value); ++- } ++- }; +++ new ThreadFactoryImpl("ControllerRequestExecutorThread_")); +++ ++ this.notifyService.initialize(); ++ if (StringUtils.isEmpty(this.controllerConfig.getControllerDLegerPeers())) { ++ throw new IllegalArgumentException("Attribute value controllerDLegerPeers of ControllerConfig is null or empty"); ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java ++index fa91f288e..33e4406e4 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java ++@@ -32,7 +32,6 @@ import java.util.Map; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ScheduledFuture; ++@@ -44,6 +43,7 @@ import org.apache.rocketmq.common.ControllerConfig; ++ import org.apache.rocketmq.common.ServiceThread; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.controller.Controller; ++ import org.apache.rocketmq.controller.elect.ElectPolicy; ++ import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy; ++@@ -66,11 +66,11 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.apache.rocketmq.remoting.protocol.body.SyncStateSet; ++ import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader; ++@@ -136,7 +136,7 @@ public class DLedgerController implements Controller { ++ this.dLedgerServer = new DLedgerServer(dLedgerConfig, nettyServerConfig, nettyClientConfig, channelEventListener); ++ this.dLedgerServer.registerStateMachine(this.statemachine); ++ this.dLedgerServer.getDLedgerLeaderElector().addRoleChangeHandler(this.roleHandler); ++- this.scanInactiveMasterService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_")); +++ this.scanInactiveMasterService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_")); ++ this.brokerLifecycleListeners = new ArrayList<>(); ++ } ++ ++@@ -513,7 +513,7 @@ public class DLedgerController implements Controller { ++ class RoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler { ++ ++ private final String selfId; ++- private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_")); +++ private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_")); ++ private volatile MemberState.Role currentRole = MemberState.Role.FOLLOWER; ++ ++ public RoleChangeHandler(final String selfId) { ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java ++index 2fbddb9cd..6ebb2c994 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java ++@@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ControllerConfig; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.controller.BrokerHeartbeatManager; ++ import org.apache.rocketmq.controller.helper.BrokerLifecycleListener; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++@@ -66,7 +67,7 @@ public class DefaultBrokerHeartbeatManager implements BrokerHeartbeatManager { ++ ++ @Override ++ public void initialize() { ++- this.scheduledService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_")); +++ this.scheduledService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_")); ++ this.executor = Executors.newFixedThreadPool(2, new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_executorService_")); ++ } ++ ++diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java ++index 15c65ebec..be327cffa 100644 ++--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java +++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java ++@@ -20,10 +20,7 @@ import java.util.Collections; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.RunnableFuture; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.concurrent.BasicThreadFactory; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++@@ -31,6 +28,7 @@ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.common.namesrv.NamesrvConfig; ++ import org.apache.rocketmq.common.utils.NetworkUtil; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.namesrv.kvconfig.KVConfigManager; ++@@ -62,10 +60,10 @@ public class NamesrvController { ++ private final NettyServerConfig nettyServerConfig; ++ private final NettyClientConfig nettyClientConfig; ++ ++- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new BasicThreadFactory.Builder().namingPattern("NSScheduledThread").daemon(true).build()); ++ ++- private final ScheduledExecutorService scanExecutorService = new ScheduledThreadPoolExecutor(1, +++ private final ScheduledExecutorService scanExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new BasicThreadFactory.Builder().namingPattern("NSScanScheduledThread").daemon(true).build()); ++ ++ private final KVConfigManager kvConfigManager; ++@@ -138,20 +136,10 @@ public class NamesrvController { ++ ++ private void initiateThreadExecutors() { ++ this.defaultThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getDefaultThreadPoolQueueCapacity()); ++- this.defaultExecutor = new ThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_")) { ++- @Override ++- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { ++- return new FutureTaskExt<>(runnable, value); ++- } ++- }; +++ this.defaultExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_")); ++ ++ this.clientRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getClientRequestThreadPoolQueueCapacity()); ++- this.clientRequestExecutor = new ThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_")) { ++- @Override ++- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { ++- return new FutureTaskExt<>(runnable, value); ++- } ++- }; +++ this.clientRequestExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_")); ++ } ++ ++ private void initiateSslContext() { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java ++index 14330dd8d..a18cf7600 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java ++@@ -21,13 +21,13 @@ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicLong; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++-import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; +++import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; ++@@ -43,7 +43,7 @@ public class GrpcChannelManager implements StartAndShutdown { ++ protected final AtomicLong nonceIdGenerator = new AtomicLong(0); ++ protected final ConcurrentMap resultNonceFutureMap = new ConcurrentHashMap<>(); ++ ++- protected final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( +++ protected final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("GrpcChannelManager_") ++ ); ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++index bcc9edd09..fe07090d5 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++@@ -22,7 +22,6 @@ import io.netty.channel.Channel; ++ import java.util.List; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.CompletableFuture; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++@@ -33,6 +32,7 @@ import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.common.thread.ThreadPoolMonitor; ++ import org.apache.rocketmq.common.thread.ThreadPoolStatusMonitor; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++@@ -178,7 +178,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu ++ new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInDefaultQueue()) ++ ); ++ ++- this.timerExecutor = Executors.newSingleThreadScheduledExecutor( +++ this.timerExecutor = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryBuilder().setNameFormat("RemotingServerScheduler-%d").build() ++ ); ++ this.timerExecutor.scheduleAtFixedRate(this::cleanExpireRequest, 10, 10, TimeUnit.SECONDS); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java ++index d2ddfc352..9786cec55 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.proxy.service; ++ ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.client.ClientChannelInfo; ++@@ -27,23 +26,24 @@ import org.apache.rocketmq.broker.client.ProducerChangeListener; ++ import org.apache.rocketmq.broker.client.ProducerGroupEvent; ++ import org.apache.rocketmq.broker.client.ProducerManager; ++ import org.apache.rocketmq.client.common.NameserverAccessConfig; +++import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor; +++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++-import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; ++ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.service.admin.AdminService; ++ import org.apache.rocketmq.proxy.service.admin.DefaultAdminService; ++ import org.apache.rocketmq.proxy.service.client.ClusterConsumerManager; +++import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor; ++ import org.apache.rocketmq.proxy.service.message.ClusterMessageService; ++ import org.apache.rocketmq.proxy.service.message.MessageService; ++ import org.apache.rocketmq.proxy.service.metadata.ClusterMetadataService; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++-import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor; ++-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++-import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor; ++ import org.apache.rocketmq.proxy.service.relay.ClusterProxyRelayService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ClusterTopicRouteService; ++@@ -73,7 +73,7 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S ++ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); ++ NameserverAccessConfig nameserverAccessConfig = new NameserverAccessConfig(proxyConfig.getNamesrvAddr(), ++ proxyConfig.getNamesrvDomain(), proxyConfig.getNamesrvDomainSubgroup()); ++- this.scheduledExecutorService = Executors.newScheduledThreadPool(3); +++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(3); ++ ++ this.messagingClientAPIFactory = new MQClientAPIFactory( ++ nameserverAccessConfig, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java ++index 4d1ca7b66..59cd92685 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.proxy.service; ++ ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++@@ -28,6 +27,7 @@ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.service.admin.AdminService; ++@@ -58,7 +58,7 @@ public class LocalServiceManager extends AbstractStartAndShutdown implements Ser ++ private final MQClientAPIFactory mqClientAPIFactory; ++ private final ChannelManager channelManager; ++ ++- private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( +++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("LocalServiceManagerScheduledThread")); ++ ++ public LocalServiceManager(BrokerController brokerController, RPCHook rpcHook) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java ++index 69f44344a..207603fe8 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java ++@@ -24,7 +24,6 @@ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++@@ -42,20 +41,21 @@ import org.apache.rocketmq.common.thread.ThreadPoolMonitor; ++ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; ++ import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++-import org.apache.rocketmq.proxy.common.RenewEvent; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.ReceiptHandleGroup; +++import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey; +++import org.apache.rocketmq.proxy.common.RenewEvent; ++ import org.apache.rocketmq.proxy.common.RenewStrategyPolicy; ++ import org.apache.rocketmq.proxy.common.channel.ChannelHelper; ++ import org.apache.rocketmq.proxy.common.utils.ExceptionUtils; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++-import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.remoting.protocol.subscription.RetryPolicy; ++ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; ++@@ -68,7 +68,7 @@ public class DefaultReceiptHandleManager extends AbstractStartAndShutdown implem ++ protected final StateEventListener eventListener; ++ protected final static RetryPolicy RENEW_POLICY = new RenewStrategyPolicy(); ++ protected final ScheduledExecutorService scheduledExecutorService = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_")); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_")); ++ protected final ThreadPoolExecutor renewalWorkerService; ++ ++ public DefaultReceiptHandleManager(MetadataService metadataService, ConsumerManager consumerManager, StateEventListener eventListener) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++index caf62a1e0..ccf094c03 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++@@ -19,25 +19,24 @@ package org.apache.rocketmq.proxy.service.route; ++ import com.github.benmanes.caffeine.cache.CacheLoader; ++ import com.github.benmanes.caffeine.cache.Caffeine; ++ import com.github.benmanes.caffeine.cache.LoadingCache; +++import com.google.common.base.Optional; ++ import java.time.Duration; ++ import java.util.List; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++- ++-import com.google.common.base.Optional; ++ import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.exception.MQClientException; +++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.client.latency.MQFaultStrategy; ++ import org.apache.rocketmq.client.latency.Resolver; ++ import org.apache.rocketmq.client.latency.ServiceDetector; ++-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.common.thread.ThreadPoolMonitor; ++ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.proxy.common.Address; ++@@ -63,7 +62,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) { ++ ProxyConfig config = ConfigurationManager.getProxyConfig(); ++ ++- this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( +++ this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("TopicRouteService_") ++ ); ++ this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor( ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++index 8491f4354..64621dd6c 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++@@ -61,7 +61,6 @@ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.concurrent.atomic.AtomicReference; ++@@ -71,6 +70,7 @@ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.ChannelEventListener; ++@@ -142,7 +142,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ ++ this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactoryImpl("NettyClientPublicExecutor_")); ++ ++- this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, +++ this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, ++ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("NettyClientScan_thread_")); ++ ++ if (eventLoopGroup != null) { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++index e626260c9..aa0d46542 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++@@ -61,6 +61,7 @@ import org.apache.rocketmq.common.constant.HAProxyConstants; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.utils.BinaryUtil; ++ import org.apache.rocketmq.common.utils.NetworkUtil; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.ChannelEventListener; ++@@ -83,7 +84,6 @@ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ ++@@ -171,7 +171,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ } ++ ++ private ScheduledExecutorService buildScheduleExecutor() { ++- return new ScheduledThreadPoolExecutor(1, +++ return ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("NettyServerScheduler_", true), ++ new ThreadPoolExecutor.DiscardOldestPolicy()); ++ } ++diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++index f2a54ddf6..02ea47f13 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++@@ -48,7 +48,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++@@ -83,6 +82,7 @@ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.common.utils.CleanupPolicyUtils; ++ import org.apache.rocketmq.common.utils.QueueTypeUtils; ++ import org.apache.rocketmq.common.utils.ServiceProvider; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; ++@@ -205,7 +205,7 @@ public class DefaultMessageStore implements MessageStore { ++ private ConcurrentMap topicConfigTable; ++ ++ private final ScheduledExecutorService scheduledCleanQueueExecutorService = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); ++ ++ public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, ++ final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws IOException { ++@@ -253,7 +253,7 @@ public class DefaultMessageStore implements MessageStore { ++ this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog()); ++ ++ this.scheduledExecutorService = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); ++ ++ this.dispatcherList = new LinkedList<>(); ++ this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue()); ++@@ -2915,7 +2915,7 @@ public class DefaultMessageStore implements MessageStore { ++ private final ExecutorService batchDispatchRequestExecutor; ++ ++ public MainBatchDispatchRequestService() { ++- batchDispatchRequestExecutor = new ThreadPoolExecutor( +++ batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor( ++ DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), ++ DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), ++ 1000 * 60, ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++index d5393fdca..f20bc3e28 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++@@ -17,10 +17,26 @@ ++ ++ package org.apache.rocketmq.store.ha.autoswitch; ++ ++- +++import java.io.IOException; +++import java.nio.channels.SocketChannel; +++import java.util.ArrayList; +++import java.util.HashSet; +++import java.util.Iterator; +++import java.util.List; +++import java.util.Map; +++import java.util.Objects; +++import java.util.Set; +++import java.util.concurrent.ConcurrentHashMap; +++import java.util.concurrent.ExecutorService; +++import java.util.concurrent.locks.Lock; +++import java.util.concurrent.locks.ReadWriteLock; +++import java.util.concurrent.locks.ReentrantReadWriteLock; +++import java.util.function.Consumer; +++import java.util.stream.Collectors; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.EpochEntry; ++@@ -36,30 +52,12 @@ import org.apache.rocketmq.store.ha.HAClient; ++ import org.apache.rocketmq.store.ha.HAConnection; ++ import org.apache.rocketmq.store.ha.HAConnectionStateNotificationService; ++ ++-import java.io.IOException; ++-import java.nio.channels.SocketChannel; ++-import java.util.ArrayList; ++-import java.util.HashSet; ++-import java.util.List; ++-import java.util.Iterator; ++-import java.util.Map; ++-import java.util.Objects; ++-import java.util.Set; ++-import java.util.concurrent.ConcurrentHashMap; ++-import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++-import java.util.concurrent.locks.Lock; ++-import java.util.concurrent.locks.ReadWriteLock; ++-import java.util.concurrent.locks.ReentrantReadWriteLock; ++-import java.util.function.Consumer; ++-import java.util.stream.Collectors; ++- ++ /** ++ * SwitchAble ha service, support switch role to master or slave. ++ */ ++ public class AutoSwitchHAService extends DefaultHAService { ++ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++- private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_")); +++ private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_")); ++ private final ConcurrentHashMap connectionCaughtUpTimeTable = new ConcurrentHashMap<>(); ++ private final List>> syncStateSetChangedListeners = new ArrayList<>(); ++ private final Set syncStateSet = new HashSet<>(); ++diff --git a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java ++index b37c90726..639084fa2 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java ++@@ -16,17 +16,25 @@ ++ */ ++ package org.apache.rocketmq.store.kv; ++ ++-import java.util.Random; +++import java.io.File; +++import java.io.IOException; +++import java.nio.file.Files; +++import java.nio.file.Paths; ++ import java.util.Iterator; ++ import java.util.List; ++ import java.util.Map; ++ import java.util.Objects; ++ import java.util.Optional; +++import java.util.Random; +++import java.util.concurrent.ConcurrentHashMap; +++import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.TopicConfig; ++ import org.apache.rocketmq.common.attribute.CleanupPolicy; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.utils.CleanupPolicyUtils; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.store.DefaultMessageStore; ++@@ -35,15 +43,6 @@ import org.apache.rocketmq.store.GetMessageResult; ++ import org.apache.rocketmq.store.SelectMappedBufferResult; ++ import org.apache.rocketmq.store.config.MessageStoreConfig; ++ ++-import java.io.File; ++-import java.io.IOException; ++-import java.nio.file.Files; ++-import java.nio.file.Paths; ++-import java.util.concurrent.ConcurrentHashMap; ++-import java.util.concurrent.Executors; ++-import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.TimeUnit; ++- ++ public class CompactionStore { ++ ++ public static final String COMPACTION_DIR = "compaction"; ++@@ -76,7 +75,7 @@ public class CompactionStore { ++ this.positionMgr = new CompactionPositionMgr(compactionPath); ++ this.compactionThreadNum = Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, config.getCompactionThreadNum())); ++ ++- this.compactionSchedule = Executors.newScheduledThreadPool(this.compactionThreadNum, +++ this.compactionSchedule = ThreadUtils.newScheduledThreadPool(this.compactionThreadNum, ++ new ThreadFactoryImpl("compactionSchedule_")); ++ this.offsetMapSize = config.getMaxOffsetMapSize() / compactionThreadNum; ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java ++index 8d38503b3..d03d15d65 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java ++@@ -24,7 +24,6 @@ import java.util.concurrent.CountDownLatch; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.FutureTask; ++ import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.TopicConfig; ++@@ -34,6 +33,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.common.utils.QueueTypeUtils; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.store.CommitLog; ++@@ -175,7 +175,7 @@ public class ConsumeQueueStore { ++ } ++ ++ private ExecutorService buildExecutorService(BlockingQueue blockingQueue, String threadNamePrefix) { ++- return new ThreadPoolExecutor( +++ return ThreadUtils.newThreadPoolExecutor( ++ this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(), ++ this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(), ++ 1000 * 60, ++diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java ++index 2dd3fc5b5..489d7b4fb 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java +++++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java ++@@ -17,7 +17,6 @@ ++ package org.apache.rocketmq.store.stats; ++ ++ import java.util.HashMap; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import org.apache.commons.lang3.tuple.Pair; ++ import org.apache.rocketmq.common.BrokerConfig; ++@@ -32,13 +31,14 @@ import org.apache.rocketmq.common.statistics.StatisticsItemScheduledPrinter; ++ import org.apache.rocketmq.common.statistics.StatisticsItemStateGetter; ++ import org.apache.rocketmq.common.statistics.StatisticsKindMeta; ++ import org.apache.rocketmq.common.statistics.StatisticsManager; +++import org.apache.rocketmq.common.stats.MomentStatsItemSet; ++ import org.apache.rocketmq.common.stats.Stats; +++import org.apache.rocketmq.common.stats.StatsItem; +++import org.apache.rocketmq.common.stats.StatsItemSet; ++ import org.apache.rocketmq.common.topic.TopicValidator; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++-import org.apache.rocketmq.common.stats.MomentStatsItemSet; ++-import org.apache.rocketmq.common.stats.StatsItem; ++-import org.apache.rocketmq.common.stats.StatsItemSet; ++ ++ public class BrokerStatsManager { ++ ++@@ -281,11 +281,11 @@ public class BrokerStatsManager { ++ ++ private void initScheduleService() { ++ this.scheduledExecutorService = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig)); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig)); ++ this.commercialExecutor = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig)); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig)); ++ this.accountExecutor = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig)); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig)); ++ } ++ ++ public MomentStatsItemSet getMomentStatsItemSetFallSize() { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++index 181f7087a..0d50de65a 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++@@ -35,7 +35,6 @@ import java.util.Set; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ConcurrentSkipListSet; ++ import java.util.concurrent.CountDownLatch; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.LinkedBlockingDeque; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++@@ -54,6 +53,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++ import org.apache.rocketmq.common.topic.TopicValidator; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.store.ConsumeQueue; ++@@ -174,11 +174,11 @@ public class TimerMessageStore { ++ this.lastBrokerRole = storeConfig.getBrokerRole(); ++ ++ if (messageStore instanceof DefaultMessageStore) { ++- scheduler = Executors.newSingleThreadScheduledExecutor( +++ scheduler = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("TimerScheduledThread", ++ ((DefaultMessageStore) messageStore).getBrokerIdentity())); ++ } else { ++- scheduler = Executors.newSingleThreadScheduledExecutor( +++ scheduler = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("TimerScheduledThread")); ++ } ++ ++diff --git a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java ++index f3d105bc6..080b7e385 100644 ++--- a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java +++++ b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java ++@@ -28,7 +28,6 @@ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.concurrent.atomic.AtomicLong; ++- ++ import javax.annotation.Generated; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++index 6dd0e8846..65d586f43 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++@@ -20,10 +20,10 @@ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ ++ public class TieredStoreExecutor { ++ ++@@ -43,20 +43,20 @@ public class TieredStoreExecutor { ++ public static ExecutorService compactIndexFileExecutor; ++ ++ public static void init() { ++- commonScheduledExecutor = new ScheduledThreadPoolExecutor( +++ commonScheduledExecutor = ThreadUtils.newScheduledThreadPool( ++ Math.max(4, Runtime.getRuntime().availableProcessors()), ++ new ThreadFactoryImpl("TieredCommonExecutor_")); ++ ++- commitExecutor = new ScheduledThreadPoolExecutor( +++ commitExecutor = ThreadUtils.newScheduledThreadPool( ++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++ new ThreadFactoryImpl("TieredCommitExecutor_")); ++ ++- cleanExpiredFileExecutor = new ScheduledThreadPoolExecutor( +++ cleanExpiredFileExecutor = ThreadUtils.newScheduledThreadPool( ++ Math.max(4, Runtime.getRuntime().availableProcessors()), ++ new ThreadFactoryImpl("TieredCleanFileExecutor_")); ++ ++ dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++- dispatchExecutor = new ThreadPoolExecutor( +++ dispatchExecutor = ThreadUtils.newThreadPoolExecutor( ++ Math.max(2, Runtime.getRuntime().availableProcessors()), ++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++ 1000 * 60, ++@@ -66,7 +66,7 @@ public class TieredStoreExecutor { ++ new ThreadPoolExecutor.DiscardOldestPolicy()); ++ ++ fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++- fetchDataExecutor = new ThreadPoolExecutor( +++ fetchDataExecutor = ThreadUtils.newThreadPoolExecutor( ++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++ Math.max(64, Runtime.getRuntime().availableProcessors() * 8), ++ 1000 * 60, ++@@ -75,7 +75,7 @@ public class TieredStoreExecutor { ++ new ThreadFactoryImpl("TieredFetchExecutor_")); ++ ++ compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++- compactIndexFileExecutor = new ThreadPoolExecutor( +++ compactIndexFileExecutor = ThreadUtils.newThreadPoolExecutor( ++ 1, ++ 1, ++ 1000 * 60, ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java ++index fa3596d51..1ebff6d8a 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java ++@@ -66,6 +66,7 @@ import org.apache.rocketmq.common.namesrv.NamesrvUtil; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.common.utils.NetworkUtil; ++ import org.apache.rocketmq.common.BoundaryType; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.RPCHook; ++@@ -193,7 +194,7 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { ++ ++ int threadPoolCoreSize = Integer.parseInt(System.getProperty("rocketmq.admin.threadpool.coresize", "20")); ++ ++- this.threadPoolExecutor = new ThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_")); +++ this.threadPoolExecutor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_")); ++ ++ break; ++ case RUNNING: ++-- ++2.32.0.windows.2 ++ ++ ++From dad6b4dadfec7a58e78a6715ec16c2eb6b17ff27 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Mon, 11 Sep 2023 14:34:10 +0800 ++Subject: [PATCH 2/6] [ISSUE #7334] `registerIncrementBrokerData` for single ++ topic update (#7335) ++ ++Signed-off-by: Ziy1-Tan ++--- ++ .../broker/topic/TopicConfigManager.java | 30 +++++++++++++++---- ++ 1 file changed, 25 insertions(+), 5 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++index 4e3c1736c..754605438 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++@@ -290,7 +290,11 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ ++ return topicConfig; ++@@ -394,7 +398,11 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ ++ return topicConfig; ++@@ -435,7 +443,11 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ ++ return topicConfig; ++@@ -461,7 +473,11 @@ public class TopicConfigManager extends ConfigManager { ++ dataVersion.nextVersion(stateMachineVersion); ++ ++ this.persist(); ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ } ++ ++@@ -484,7 +500,11 @@ public class TopicConfigManager extends ConfigManager { ++ dataVersion.nextVersion(stateMachineVersion); ++ ++ this.persist(); ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ } ++ ++-- ++2.32.0.windows.2 ++ ++ ++From 0dbd0772b99f618f757d42cd64542b83e2100e4f Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Mon, 11 Sep 2023 15:48:07 +0800 ++Subject: [PATCH 3/6] [ISSUE #7326] Split the request to register to the ++ nameserver (#7325) ++ ++Signed-off-by: Ziy1-Tan ++--- ++ .../rocketmq/broker/BrokerController.java | 41 +++++++++++-------- ++ .../broker/topic/TopicConfigManager.java | 21 ++++++++++ ++ .../apache/rocketmq/common/BrokerConfig.java | 24 +++++++++++ ++ .../test/route/CreateAndUpdateTopicIT.java | 31 ++++++++++++++ ++ 4 files changed, 99 insertions(+), 18 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 275b64b1a..9e49f636d 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1765,29 +1765,34 @@ public class BrokerController { ++ } ++ ++ public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) { +++ ConcurrentMap topicConfigMap = this.getTopicConfigManager().getTopicConfigTable(); +++ ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); ++ ++- TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper(); ++- ++- topicConfigWrapper.setDataVersion(this.getTopicConfigManager().getDataVersion()); ++- topicConfigWrapper.setTopicConfigTable(this.getTopicConfigManager().getTopicConfigTable()); ++- ++- topicConfigWrapper.setTopicQueueMappingInfoMap(this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream().map( ++- entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue())) ++- ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); ++- ++- if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) ++- || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { ++- ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); ++- for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) { ++- TopicConfig tmp = +++ for (TopicConfig topicConfig : topicConfigMap.values()) { +++ if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) +++ || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { +++ topicConfigTable.put(topicConfig.getTopicName(), ++ new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), ++- topicConfig.getPerm() & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); ++- topicConfigTable.put(topicConfig.getTopicName(), tmp); +++ topicConfig.getPerm() & getBrokerConfig().getBrokerPermission())); +++ } else { +++ topicConfigTable.put(topicConfig.getTopicName(), topicConfig); +++ } +++ +++ if (this.brokerConfig.isEnableSplitRegistration() +++ && topicConfigTable.size() >= this.brokerConfig.getSplitRegistrationSize()) { +++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildSerializeWrapper(topicConfigTable); +++ doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper); +++ topicConfigTable.clear(); ++ } ++- topicConfigWrapper.setTopicConfigTable(topicConfigTable); ++ } ++ ++- if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), +++ Map topicQueueMappingInfoMap = this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream() +++ .map(entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue()))) +++ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); +++ +++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager(). +++ buildSerializeWrapper(topicConfigTable, topicQueueMappingInfoMap); +++ if (this.brokerConfig.isEnableSplitRegistration() || forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), ++ this.getBrokerAddr(), ++ this.brokerConfig.getBrokerName(), ++ this.brokerConfig.getBrokerId(), ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++index 754605438..8537929be 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++@@ -29,6 +29,7 @@ import java.util.concurrent.locks.ReentrantLock; ++ ++ import com.google.common.collect.ImmutableMap; ++ +++import com.google.common.collect.Maps; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.broker.BrokerPathConfigHelper; ++@@ -47,7 +48,9 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.DataVersion; ++ import org.apache.rocketmq.remoting.protocol.body.KVTable; +++import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper; ++ import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; +++import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo; ++ ++ import static com.google.common.base.Preconditions.checkNotNull; ++ ++@@ -609,6 +612,24 @@ public class TopicConfigManager extends ConfigManager { ++ return topicConfigSerializeWrapper; ++ } ++ +++ public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper(final ConcurrentMap topicConfigTable) { +++ return buildSerializeWrapper(topicConfigTable, Maps.newHashMap()); +++ } +++ +++ public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper( +++ final ConcurrentMap topicConfigTable, +++ final Map topicQueueMappingInfoMap +++ ) { +++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper(); +++ topicConfigWrapper.setTopicConfigTable(topicConfigTable); +++ topicConfigWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap); +++ topicConfigWrapper.setDataVersion(this.getDataVersion()); +++ if (this.brokerController.getBrokerConfig().isEnableSplitRegistration()) { +++ this.getDataVersion().nextVersion(); +++ } +++ return topicConfigWrapper; +++ } +++ ++ @Override ++ public String encode() { ++ return encode(false); ++diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++index 45d26b29c..0d248c4e1 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++@@ -396,6 +396,14 @@ public class BrokerConfig extends BrokerIdentity { ++ ++ private boolean enableMixedMessageType = false; ++ +++ /** +++ * This flag and deleteTopicWithBrokerRegistration flag in the NameServer cannot be set to true at the same time, +++ * otherwise there will be a loss of routing +++ */ +++ private boolean enableSplitRegistration = false; +++ +++ private int splitRegistrationSize = 800; +++ ++ public long getMaxPopPollingSize() { ++ return maxPopPollingSize; ++ } ++@@ -1731,4 +1739,20 @@ public class BrokerConfig extends BrokerIdentity { ++ public void setEnableMixedMessageType(boolean enableMixedMessageType) { ++ this.enableMixedMessageType = enableMixedMessageType; ++ } +++ +++ public boolean isEnableSplitRegistration() { +++ return enableSplitRegistration; +++ } +++ +++ public void setEnableSplitRegistration(boolean enableSplitRegistration) { +++ this.enableSplitRegistration = enableSplitRegistration; +++ } +++ +++ public int getSplitRegistrationSize() { +++ return splitRegistrationSize; +++ } +++ +++ public void setSplitRegistrationSize(int splitRegistrationSize) { +++ this.splitRegistrationSize = splitRegistrationSize; +++ } ++ } ++diff --git a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java ++index 7e3c7b871..2370e68c0 100644 ++--- a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java +++++ b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java ++@@ -17,6 +17,7 @@ ++ ++ package org.apache.rocketmq.test.route; ++ +++import org.apache.rocketmq.common.TopicConfig; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++ import org.apache.rocketmq.test.base.BaseConf; ++ import org.apache.rocketmq.test.util.MQAdminTestUtils; ++@@ -111,4 +112,34 @@ public class CreateAndUpdateTopicIT extends BaseConf { ++ brokerController3.getBrokerConfig().setEnableSingleTopicRegister(false); ++ namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(false); ++ } +++ +++ @Test +++ public void testCreateOrUpdateTopic_EnableSplitRegistration() { +++ brokerController1.getBrokerConfig().setEnableSplitRegistration(true); +++ brokerController2.getBrokerConfig().setEnableSplitRegistration(true); +++ brokerController3.getBrokerConfig().setEnableSplitRegistration(true); +++ +++ String testTopic = "test-topic-"; +++ +++ for (int i = 0; i < 1000; i++) { +++ TopicConfig topicConfig = new TopicConfig(testTopic + i, 8, 8); +++ brokerController1.getTopicConfigManager().updateTopicConfig(topicConfig); +++ brokerController2.getTopicConfigManager().updateTopicConfig(topicConfig); +++ brokerController3.getTopicConfigManager().updateTopicConfig(topicConfig); +++ } +++ +++ brokerController1.registerBrokerAll(false, true, true); +++ brokerController2.registerBrokerAll(false, true, true); +++ brokerController3.registerBrokerAll(false, true, true); +++ +++ for (int i = 0; i < 1000; i++) { +++ TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic + i); +++ assertThat(route.getBrokerDatas()).hasSize(3); +++ assertThat(route.getQueueDatas()).hasSize(3); +++ } +++ +++ brokerController1.getBrokerConfig().setEnableSplitRegistration(false); +++ brokerController2.getBrokerConfig().setEnableSplitRegistration(false); +++ brokerController3.getBrokerConfig().setEnableSplitRegistration(false); +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From a9e353285cea762b0c5eab567bdfa8e5c8c2d279 Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Mon, 11 Sep 2023 15:55:18 +0800 ++Subject: [PATCH 4/6] Add the configuration of topicQueueLock number to better ++ support different scenarios (#7317) ++ ++--- ++ .../main/java/org/apache/rocketmq/store/CommitLog.java | 2 +- ++ .../java/org/apache/rocketmq/store/TopicQueueLock.java | 8 ++++++++ ++ .../rocketmq/store/config/MessageStoreConfig.java | 10 ++++++++++ ++ 3 files changed, 19 insertions(+), 1 deletion(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++index e6ee3bacc..456bf2b86 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++@@ -122,7 +122,7 @@ public class CommitLog implements Swappable { ++ ++ this.flushDiskWatcher = new FlushDiskWatcher(); ++ ++- this.topicQueueLock = new TopicQueueLock(); +++ this.topicQueueLock = new TopicQueueLock(messageStore.getMessageStoreConfig().getTopicQueueLockNum()); ++ ++ this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); ++ } ++diff --git a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java ++index a78eeed23..5a131b5c3 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java +++++ b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java ++@@ -34,6 +34,14 @@ public class TopicQueueLock { ++ } ++ } ++ +++ public TopicQueueLock(int size) { +++ this.size = size; +++ this.lockList = new ArrayList<>(size); +++ for (int i = 0; i < this.size; i++) { +++ this.lockList.add(new ReentrantLock()); +++ } +++ } +++ ++ public void lock(String topicQueueKey) { ++ Lock lock = this.lockList.get((topicQueueKey.hashCode() & 0x7fffffff) % this.size); ++ lock.lock(); ++diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java ++index efb728ac0..9fa448043 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java ++@@ -401,6 +401,8 @@ public class MessageStoreConfig { ++ private long memTableFlushInterval = 60 * 60 * 1000L; ++ private boolean enableRocksDBLog = false; ++ +++ private int topicQueueLockNum = 32; +++ ++ public boolean isDebugLockEnable() { ++ return debugLockEnable; ++ } ++@@ -1751,4 +1753,12 @@ public class MessageStoreConfig { ++ public void setEnableRocksDBLog(boolean enableRocksDBLog) { ++ this.enableRocksDBLog = enableRocksDBLog; ++ } +++ +++ public int getTopicQueueLockNum() { +++ return topicQueueLockNum; +++ } +++ +++ public void setTopicQueueLockNum(int topicQueueLockNum) { +++ this.topicQueueLockNum = topicQueueLockNum; +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 57f04c95d3a2ba6b91583058a6e4eda209f72d6e Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Mon, 11 Sep 2023 18:23:25 +0800 ++Subject: [PATCH 5/6] [ISSUE #7343] Rollback modifications to registerProcessor ++ ++Co-authored-by: guyinyou ++--- ++ .../java/org/apache/rocketmq/broker/BrokerController.java | 4 ++-- ++ 1 file changed, 2 insertions(+), 2 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 9e49f636d..13a3feb4e 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -827,6 +827,8 @@ public class BrokerController { ++ ++ initializeResources(); ++ +++ registerProcessor(); +++ ++ initializeScheduledTasks(); ++ ++ initialTransaction(); ++@@ -1687,8 +1689,6 @@ public class BrokerController { ++ } ++ } ++ }, 10, 5, TimeUnit.SECONDS); ++- ++- registerProcessor(); ++ } ++ ++ protected void scheduleSendHeartbeat() { ++-- ++2.32.0.windows.2 ++ ++ ++From dad6ad09d13dadc36b6342671c77f619bbb8c522 Mon Sep 17 00:00:00 2001 ++From: Ao Qiao ++Date: Tue, 12 Sep 2023 08:28:45 +0800 ++Subject: [PATCH 6/6] [ISSUE #7340] Abstract Duplicate code into a method in ++ `TopicConfigManager` (#7341) ++ ++--- ++ .../broker/topic/TopicConfigManager.java | 44 ++++++------------- ++ 1 file changed, 14 insertions(+), 30 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++index 8537929be..511d29e12 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++@@ -293,11 +293,7 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ ++ return topicConfig; ++@@ -337,11 +333,7 @@ public class TopicConfigManager extends ConfigManager { ++ log.error("createTopicIfAbsent ", e); ++ } ++ if (createNew && register) { ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ return getTopicConfig(topicConfig.getTopicName()); ++ } ++@@ -401,11 +393,7 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ ++ return topicConfig; ++@@ -446,11 +434,7 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ ++ return topicConfig; ++@@ -476,11 +460,7 @@ public class TopicConfigManager extends ConfigManager { ++ dataVersion.nextVersion(stateMachineVersion); ++ ++ this.persist(); ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ } ++ ++@@ -503,11 +483,7 @@ public class TopicConfigManager extends ConfigManager { ++ dataVersion.nextVersion(stateMachineVersion); ++ ++ this.persist(); ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ } ++ ++@@ -699,6 +675,14 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ } ++ +++ private void registerBrokerData(TopicConfig topicConfig) { +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } +++ } +++ ++ public boolean containsTopic(String topic) { ++ return topicConfigTable.containsKey(topic); ++ } ++-- ++2.32.0.windows.2 ++ +diff --git a/patch018-backport-enhancement-of-tiered-storage.patch b/patch018-backport-enhancement-of-tiered-storage.patch +new file mode 100644 +index 000000000..ed79df9c7 +--- /dev/null ++++ b/patch018-backport-enhancement-of-tiered-storage.patch +@@ -0,0 +1,601 @@ ++From 1a8e7cb17cb29ed33b0196b52e452a6e76ade781 Mon Sep 17 00:00:00 2001 ++From: yuz10 <845238369@qq.com> ++Date: Tue, 12 Sep 2023 19:33:41 +0800 ++Subject: [PATCH 1/5] [ISSUE #7345] Fix wrong result of searchOffset in tiered ++ storage ++ ++--- ++ .../tieredstore/file/TieredFlatFile.java | 5 +- ++ .../tieredstore/file/TieredFlatFileTest.java | 46 +++++++++++++++++-- ++ 2 files changed, 46 insertions(+), 5 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++index 426c4e09d..d973179ee 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++@@ -365,7 +365,10 @@ public class TieredFlatFile { ++ if (!segmentList.isEmpty()) { ++ return boundaryType == BoundaryType.UPPER ? segmentList.get(0) : segmentList.get(segmentList.size() - 1); ++ } ++- return fileSegmentList.isEmpty() ? null : fileSegmentList.get(fileSegmentList.size() - 1); +++ if (fileSegmentList.isEmpty()) { +++ return null; +++ } +++ return boundaryType == BoundaryType.UPPER ? fileSegmentList.get(fileSegmentList.size() - 1) : fileSegmentList.get(0); ++ } finally { ++ fileSegmentLock.readLock().unlock(); ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++index 7a4d05969..7e2fbf201 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++@@ -16,10 +16,7 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ ++-import java.io.IOException; ++-import java.nio.ByteBuffer; ++-import java.util.ArrayList; ++-import java.util.List; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++@@ -35,6 +32,11 @@ import org.junit.Assert; ++ import org.junit.Before; ++ import org.junit.Test; ++ +++import java.io.IOException; +++import java.nio.ByteBuffer; +++import java.util.ArrayList; +++import java.util.List; +++ ++ public class TieredFlatFileTest { ++ ++ private final String storePath = TieredStoreTestUtil.getRandomStorePath(); ++@@ -301,4 +303,40 @@ public class TieredFlatFileTest { ++ fileQueue.rollingNewFile(); ++ Assert.assertEquals(2, fileQueue.getFileSegmentCount()); ++ } +++ +++ @Test +++ public void testGetFileByTime() { +++ String filePath = TieredStoreUtil.toPath(queue); +++ TieredFlatFile tieredFlatFile = fileQueueFactory.createFlatFileForCommitLog(filePath); +++ TieredFileSegment fileSegment1 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig); +++ fileSegment1.setMinTimestamp(100); +++ fileSegment1.setMaxTimestamp(200); +++ +++ TieredFileSegment fileSegment2 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig); +++ fileSegment2.setMinTimestamp(200); +++ fileSegment2.setMaxTimestamp(300); +++ +++ tieredFlatFile.getFileSegmentList().add(fileSegment1); +++ tieredFlatFile.getFileSegmentList().add(fileSegment2); +++ +++ TieredFileSegment segmentUpper = tieredFlatFile.getFileByTime(400, BoundaryType.UPPER); +++ Assert.assertEquals(fileSegment2, segmentUpper); +++ +++ TieredFileSegment segmentLower = tieredFlatFile.getFileByTime(400, BoundaryType.LOWER); +++ Assert.assertEquals(fileSegment2, segmentLower); +++ +++ +++ TieredFileSegment segmentUpper2 = tieredFlatFile.getFileByTime(0, BoundaryType.UPPER); +++ Assert.assertEquals(fileSegment1, segmentUpper2); +++ +++ TieredFileSegment segmentLower2 = tieredFlatFile.getFileByTime(0, BoundaryType.LOWER); +++ Assert.assertEquals(fileSegment1, segmentLower2); +++ +++ +++ TieredFileSegment segmentUpper3 = tieredFlatFile.getFileByTime(200, BoundaryType.UPPER); +++ Assert.assertEquals(fileSegment1, segmentUpper3); +++ +++ TieredFileSegment segmentLower3 = tieredFlatFile.getFileByTime(200, BoundaryType.LOWER); +++ Assert.assertEquals(fileSegment2, segmentLower3); +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From fd32dae2ab59f86dd215eeec405bf4fa6212bcb3 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Tue, 12 Sep 2023 19:58:08 +0800 ++Subject: [PATCH 2/5] [ISSUE #6633] Not clear uninitialized files and fix ++ metadata recover (#7342) ++ ++--- ++ .../tieredstore/file/TieredFlatFile.java | 53 +++++++------------ ++ .../file/TieredFlatFileManager.java | 10 ++-- ++ 2 files changed, 22 insertions(+), 41 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++index d973179ee..d96eb6e8f 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ ++-import com.alibaba.fastjson.JSON; ++ import com.google.common.annotations.VisibleForTesting; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; ++@@ -25,13 +24,13 @@ import java.util.Comparator; ++ import java.util.HashSet; ++ import java.util.LinkedList; ++ import java.util.List; ++-import java.util.Objects; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.CopyOnWriteArrayList; ++ import java.util.concurrent.locks.ReentrantReadWriteLock; ++ import java.util.stream.Collectors; ++ import javax.annotation.Nullable; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.tieredstore.common.AppendResult; ++@@ -43,7 +42,6 @@ import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore; ++ import org.apache.rocketmq.tieredstore.provider.FileSegmentAllocator; ++ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++-import org.apache.rocketmq.common.BoundaryType; ++ ++ public class TieredFlatFile { ++ ++@@ -177,7 +175,10 @@ public class TieredFlatFile { ++ } ++ } ++ ++- private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fileSegment) { +++ /** +++ * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full +++ */ +++ public void updateFileSegment(TieredFileSegment fileSegment) { ++ ++ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( ++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); ++@@ -186,45 +187,24 @@ public class TieredFlatFile { ++ if (metadata == null) { ++ metadata = new FileSegmentMetadata( ++ this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); ++- metadata.setCreateTimestamp(fileSegment.getMinTimestamp()); ++- metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); ++- metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++- if (fileSegment.isClosed()) { ++- metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); ++- } ++- this.tieredMetadataStore.updateFileSegment(metadata); +++ metadata.setCreateTimestamp(System.currentTimeMillis()); ++ } ++- return metadata; ++- } ++- ++- /** ++- * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full ++- */ ++- public void updateFileSegment(TieredFileSegment fileSegment) { ++- FileSegmentMetadata segmentMetadata = getOrCreateFileSegmentMetadata(fileSegment); ++ ++- if (segmentMetadata.getStatus() == FileSegmentMetadata.STATUS_NEW ++- && fileSegment.isFull() ++- && !fileSegment.needCommit()) { +++ metadata.setSize(fileSegment.getCommitPosition()); +++ metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); +++ metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++ ++- segmentMetadata.markSealed(); +++ if (fileSegment.isFull() && !fileSegment.needCommit()) { +++ if (metadata.getStatus() == FileSegmentMetadata.STATUS_NEW) { +++ metadata.markSealed(); +++ } ++ } ++ ++ if (fileSegment.isClosed()) { ++- segmentMetadata.setStatus(FileSegmentMetadata.STATUS_DELETED); +++ metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); ++ } ++ ++- segmentMetadata.setSize(fileSegment.getCommitPosition()); ++- segmentMetadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++- ++- FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( ++- this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); ++- ++- if (!Objects.equals(metadata, segmentMetadata)) { ++- this.tieredMetadataStore.updateFileSegment(segmentMetadata); ++- logger.debug("TieredFlatFile#UpdateSegmentMetadata, filePath: {}, content: {}", ++- segmentMetadata.getPath(), JSON.toJSONString(segmentMetadata)); ++- } +++ this.tieredMetadataStore.updateFileSegment(metadata); ++ } ++ ++ private void checkAndFixFileSize() { ++@@ -598,6 +578,9 @@ public class TieredFlatFile { ++ logger.error("TieredFlatFile#destroy: mark file segment: {} is deleted failed", fileSegment.getPath(), e); ++ } ++ fileSegment.destroyFile(); +++ if (!fileSegment.exists()) { +++ tieredMetadataStore.deleteFileSegment(filePath, fileType, fileSegment.getBaseOffset()); +++ } ++ } ++ fileSegmentList.clear(); ++ needCommitFileSegmentList.clear(); ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++index 7c744af3b..087ea8c9c 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++@@ -136,15 +136,13 @@ public class TieredFlatFileManager { ++ TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime()); ++ for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) { ++ TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> { ++- flatFile.getCompositeFlatFileLock().lock(); ++ try { +++ flatFile.getCompositeFlatFileLock().lock(); ++ flatFile.cleanExpiredFile(expiredTimeStamp); ++ flatFile.destroyExpiredFile(); ++- if (flatFile.getConsumeQueueBaseOffset() == -1) { ++- logger.info("Clean flatFile because file not initialized, topic={}, queueId={}", ++- flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId()); ++- destroyCompositeFile(flatFile.getMessageQueue()); ++- } +++ } catch (Throwable t) { +++ logger.error("Do Clean expired file error, topic={}, queueId={}", +++ flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), t); ++ } finally { ++ flatFile.getCompositeFlatFileLock().unlock(); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 4a8e0d5b851d1f9573cda79b7d2e42ee498809da Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Wed, 13 Sep 2023 16:08:03 +0800 ++Subject: [PATCH 3/5] [ISSUE #7351] Allow mqadmin to operate slave nodes ++ ++Co-authored-by: guyinyou ++--- ++ .../processor/AdminBrokerProcessor.java | 12 -- ++ .../processor/AdminBrokerProcessorTest.java | 106 ------------------ ++ 2 files changed, 118 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++index 8fbcd3c94..9e48431be 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++@@ -406,9 +406,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, ++ RemotingCommand request) throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- if (validateSlave(response)) { ++- return response; ++- } ++ final CreateTopicRequestHeader requestHeader = ++ (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class); ++ ++@@ -519,9 +516,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, ++ RemotingCommand request) throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- if (validateSlave(response)) { ++- return response; ++- } ++ DeleteTopicRequestHeader requestHeader = ++ (DeleteTopicRequestHeader) request.decodeCommandCustomHeader(DeleteTopicRequestHeader.class); ++ ++@@ -1413,9 +1407,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) ++ throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- if (validateSlave(response)) { ++- return response; ++- } ++ ++ LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}", ++ RemotingHelper.parseChannelRemoteAddr(ctx.channel())); ++@@ -1480,9 +1471,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx, ++ RemotingCommand request) throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- if (validateSlave(response)) { ++- return response; ++- } ++ DeleteSubscriptionGroupRequestHeader requestHeader = ++ (DeleteSubscriptionGroupRequestHeader) request.decodeCommandCustomHeader(DeleteSubscriptionGroupRequestHeader.class); ++ ++diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++index 9d17011b6..ec252cece 100644 ++--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++@@ -76,7 +76,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfi ++ import org.apache.rocketmq.store.DefaultMessageStore; ++ import org.apache.rocketmq.store.MessageStore; ++ import org.apache.rocketmq.store.SelectMappedBufferResult; ++-import org.apache.rocketmq.store.config.BrokerRole; ++ import org.apache.rocketmq.store.config.MessageStoreConfig; ++ import org.apache.rocketmq.store.logfile.DefaultMappedFile; ++ import org.apache.rocketmq.store.stats.BrokerStats; ++@@ -250,32 +249,6 @@ public class AdminBrokerProcessorTest { ++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); ++ } ++ ++- @Test ++- public void testUpdateAndCreateTopicOnSlaveInRocksdb() throws Exception { ++- if (notToBeExecuted()) { ++- return; ++- } ++- initRocksdbTopicManager(); ++- testUpdateAndCreateTopicOnSlave(); ++- } ++- ++- @Test ++- public void testUpdateAndCreateTopicOnSlave() throws Exception { ++- // setup ++- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); ++- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); ++- defaultMessageStore = mock(DefaultMessageStore.class); ++- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); ++- ++- // test on slave ++- String topic = "TEST_CREATE_TOPIC"; ++- RemotingCommand request = buildCreateTopicRequest(topic); ++- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); ++- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); ++- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + ++- "please execute it from master broker."); ++- } ++- ++ @Test ++ public void testDeleteTopicInRocksdb() throws Exception { ++ if (notToBeExecuted()) { ++@@ -301,31 +274,6 @@ public class AdminBrokerProcessorTest { ++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); ++ } ++ ++- @Test ++- public void testDeleteTopicOnSlaveInRocksdb() throws Exception { ++- if (notToBeExecuted()) { ++- return; ++- } ++- initRocksdbTopicManager(); ++- testDeleteTopicOnSlave(); ++- } ++- ++- @Test ++- public void testDeleteTopicOnSlave() throws Exception { ++- // setup ++- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); ++- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); ++- defaultMessageStore = mock(DefaultMessageStore.class); ++- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); ++- ++- String topic = "TEST_DELETE_TOPIC"; ++- RemotingCommand request = buildDeleteTopicRequest(topic); ++- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); ++- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); ++- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + ++- "please execute it from master broker."); ++- } ++- ++ @Test ++ public void testDeleteWithPopRetryTopic() throws Exception { ++ String topic = "topicA"; ++@@ -538,36 +486,6 @@ public class AdminBrokerProcessorTest { ++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); ++ } ++ ++- @Test ++- public void testUpdateAndCreateSubscriptionGroupOnSlaveInRocksdb() throws Exception { ++- initRocksdbSubscriptionManager(); ++- testUpdateAndCreateSubscriptionGroupOnSlave(); ++- } ++- ++- @Test ++- public void testUpdateAndCreateSubscriptionGroupOnSlave() throws RemotingCommandException { ++- // Setup ++- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); ++- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); ++- defaultMessageStore = mock(DefaultMessageStore.class); ++- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); ++- ++- // Test ++- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null); ++- SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); ++- subscriptionGroupConfig.setBrokerId(1); ++- subscriptionGroupConfig.setGroupName("groupId"); ++- subscriptionGroupConfig.setConsumeEnable(Boolean.TRUE); ++- subscriptionGroupConfig.setConsumeBroadcastEnable(Boolean.TRUE); ++- subscriptionGroupConfig.setRetryMaxTimes(111); ++- subscriptionGroupConfig.setConsumeFromMinEnable(Boolean.TRUE); ++- request.setBody(JSON.toJSON(subscriptionGroupConfig).toString().getBytes()); ++- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); ++- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); ++- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + ++- "please execute it from master broker."); ++- } ++- ++ @Test ++ public void testGetAllSubscriptionGroupInRocksdb() throws Exception { ++ initRocksdbSubscriptionManager(); ++@@ -596,30 +514,6 @@ public class AdminBrokerProcessorTest { ++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); ++ } ++ ++- @Test ++- public void testDeleteSubscriptionGroupOnSlaveInRocksdb() throws Exception { ++- initRocksdbSubscriptionManager(); ++- testDeleteSubscriptionGroupOnSlave(); ++- } ++- ++- @Test ++- public void testDeleteSubscriptionGroupOnSlave() throws RemotingCommandException { ++- // Setup ++- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); ++- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); ++- defaultMessageStore = mock(DefaultMessageStore.class); ++- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); ++- ++- // Test ++- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, null); ++- request.addExtField("groupName", "GID-Group-Name"); ++- request.addExtField("removeOffset", "true"); ++- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); ++- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); ++- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + ++- "please execute it from master broker."); ++- } ++- ++ @Test ++ public void testGetTopicStatsInfo() throws RemotingCommandException { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, null); ++-- ++2.32.0.windows.2 ++ ++ ++From 831fcc76cd7cd362bb6c136c287c624bb7eaf40a Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Tue, 19 Sep 2023 10:04:04 +0800 ++Subject: [PATCH 4/5] [ISSUE #7363] Fix get message from tiered storage return ++ incorrect next pull offset (#7365) ++ ++--- ++ .../tieredstore/TieredMessageFetcher.java | 2 +- ++ .../tieredstore/TieredMessageStore.java | 29 ++++++++++--------- ++ .../tieredstore/TieredMessageStoreTest.java | 5 ++-- ++ 3 files changed, 20 insertions(+), 16 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++index 766ff64f6..c948fa3fa 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++@@ -319,7 +319,7 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ } ++ ++ // if cache is miss, immediately pull messages ++- LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + +++ LOGGER.info("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + ++ "topic: {}, queue: {}, queue offset: {}, max message num: {}", ++ mq.getTopic(), mq.getQueueId(), queueOffset, maxCount); ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index 9fb1b2f01..d7d13d61e 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -147,6 +147,11 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ public CompletableFuture getMessageAsync(String group, String topic, ++ int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { ++ +++ // For system topic, force reading from local store +++ if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { +++ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); +++ } +++ ++ if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) { ++ logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); ++ } else { ++@@ -158,6 +163,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ return fetcher ++ .getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter) ++ .thenApply(result -> { +++ ++ Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder() ++ .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_GET_MESSAGE) ++ .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic) ++@@ -166,8 +172,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); ++ ++ if (result.getStatus() == GetMessageStatus.OFFSET_FOUND_NULL || ++- result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_ONE || ++- result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_BADLY) { +++ result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { ++ ++ if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) { ++ TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes); ++@@ -178,14 +183,8 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ } ++ } ++ ++- // Fetch system topic data from the broker when using the force level. ++- if (result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { ++- if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { ++- return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); ++- } ++- } ++- ++ if (result.getStatus() != GetMessageStatus.FOUND && +++ result.getStatus() != GetMessageStatus.NO_MATCHED_LOGIC_QUEUE && ++ result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_ONE && ++ result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_BADLY) { ++ logger.warn("GetMessageAsync not found and message is not in next store, result: {}, " + ++@@ -206,10 +205,14 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) { ++ result.setMinOffset(minOffsetInQueue); ++ } ++- long maxOffsetInQueue = next.getMaxOffsetInQueue(topic, queueId); ++- if (maxOffsetInQueue >= 0 && maxOffsetInQueue > result.getMaxOffset()) { ++- result.setMaxOffset(maxOffsetInQueue); ++- } +++ +++ // In general, the local cq offset is slightly greater than the commit offset in read message, +++ // so there is no need to update the maximum offset to the local cq offset here, +++ // otherwise it will cause repeated consumption after next begin offset over commit offset. +++ +++ logger.trace("GetMessageAsync result, group: {}, topic: {}, queueId: {}, offset: {}, count:{}, {}", +++ group, topic, queueId, offset, maxMsgNums, result); +++ ++ return result; ++ }).exceptionally(e -> { ++ logger.error("GetMessageAsync from tiered store failed", e); ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++index 2451199c2..07af1fc8b 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++@@ -168,7 +168,7 @@ public class TieredMessageStoreTest { ++ GetMessageResult result1 = new GetMessageResult(); ++ result1.setStatus(GetMessageStatus.FOUND); ++ GetMessageResult result2 = new GetMessageResult(); ++- result2.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING); +++ result2.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY); ++ ++ when(fetcher.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(result1)); ++ when(nextStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(result2); ++@@ -188,7 +188,8 @@ public class TieredMessageStoreTest { ++ properties.setProperty("tieredStorageLevel", "3"); ++ configuration.update(properties); ++ when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); ++- Assert.assertSame(result2, store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null)); +++ Assert.assertEquals(result2.getStatus(), +++ store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null).getStatus()); ++ } ++ ++ @Test ++-- ++2.32.0.windows.2 ++ ++ ++From f05a8da760dfade411ad56ef874f477988479cf9 Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Wed, 20 Sep 2023 15:06:21 +0800 ++Subject: [PATCH 5/5] Print admin queue watermark in log (#7372) ++ ++--- ++ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 13a3feb4e..53e2e1b62 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1182,6 +1182,7 @@ public class BrokerController { ++ LOG_WATER_MARK.info("[WATERMARK] ClientManager Queue Size: {} SlowTimeMills: {}", this.clientManagerThreadPoolQueue.size(), this.headSlowTimeMills(this.clientManagerThreadPoolQueue)); ++ LOG_WATER_MARK.info("[WATERMARK] Heartbeat Queue Size: {} SlowTimeMills: {}", this.heartbeatThreadPoolQueue.size(), this.headSlowTimeMills(this.heartbeatThreadPoolQueue)); ++ LOG_WATER_MARK.info("[WATERMARK] Ack Queue Size: {} SlowTimeMills: {}", this.ackThreadPoolQueue.size(), headSlowTimeMills(this.ackThreadPoolQueue)); +++ LOG_WATER_MARK.info("[WATERMARK] Admin Queue Size: {} SlowTimeMills: {}", this.adminBrokerThreadPoolQueue.size(), headSlowTimeMills(this.adminBrokerThreadPoolQueue)); ++ } ++ ++ public MessageStore getMessageStore() { ++-- ++2.32.0.windows.2 ++ +diff --git a/patch019-backport-some-bugfix.patch b/patch019-backport-some-bugfix.patch +new file mode 100644 +index 000000000..d85f01c98 +--- /dev/null ++++ b/patch019-backport-some-bugfix.patch +@@ -0,0 +1,1499 @@ ++From 42fcd278ca84f6988d48a7d11271fc81b921d59a Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Wed, 20 Sep 2023 15:41:23 +0800 ++Subject: [PATCH 01/12] [ISSUE #7374] Prepare to release Apache RocketMQ 5.1.4 ++ (#7375) ++ ++--- ++ common/src/main/java/org/apache/rocketmq/common/MQVersion.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java ++index bfd07a895..4f1990ff8 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java ++@@ -18,7 +18,7 @@ package org.apache.rocketmq.common; ++ ++ public class MQVersion { ++ ++- public static final int CURRENT_VERSION = Version.V5_1_3.ordinal(); +++ public static final int CURRENT_VERSION = Version.V5_1_4.ordinal(); ++ ++ public static String getVersionDesc(int value) { ++ int length = Version.values().length; ++-- ++2.32.0.windows.2 ++ ++ ++From b8610d87bb55de1f4413460c05da529dab60c1c1 Mon Sep 17 00:00:00 2001 ++From: Jixiang Jin ++Date: Thu, 21 Sep 2023 16:21:44 +0800 ++Subject: [PATCH 02/12] Replace loggingMetricExporter with ++ OtlpJsonLoggingMetricExporter. (#7373) ++ ++* Replace loggingMetricExporter with OtlpJsonLoggingMetricExporter. ++ ++* Fix bazel workspace ++ ++* Move OtlpJsonLoggingMetricExporter to proxy and controller. ++ ++* Fix Bazel imports. ++--- ++ WORKSPACE | 1 + ++ broker/BUILD.bazel | 1 + ++ .../rocketmq/broker/metrics/BrokerMetricsManager.java | 9 +++++---- ++ broker/src/main/resources/rmq.broker.logback.xml | 5 +++++ ++ common/BUILD.bazel | 1 + ++ common/pom.xml | 4 ++++ ++ controller/BUILD.bazel | 1 + ++ .../controller/metrics/ControllerMetricsManager.java | 9 +++++---- ++ pom.xml | 5 +++++ ++ proxy/BUILD.bazel | 1 + ++ .../rocketmq/proxy/metrics/ProxyMetricsManager.java | 11 ++++++----- ++ proxy/src/main/resources/rmq.proxy.logback.xml | 5 +++++ ++ tieredstore/BUILD.bazel | 1 + ++ 13 files changed, 41 insertions(+), 13 deletions(-) ++ ++diff --git a/WORKSPACE b/WORKSPACE ++index 3126f2d1d..8640485ba 100644 ++--- a/WORKSPACE +++++ b/WORKSPACE ++@@ -92,6 +92,7 @@ maven_install( ++ "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha", ++ "io.opentelemetry:opentelemetry-exporter-logging:1.29.0", ++ "io.opentelemetry:opentelemetry-sdk:1.29.0", +++ "io.opentelemetry:opentelemetry-exporter-logging-otlp:1.29.0", ++ "com.squareup.okio:okio-jvm:3.0.0", ++ "io.opentelemetry:opentelemetry-api:1.29.0", ++ "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0", ++diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel ++index 6adcdc7b9..64cb1b341 100644 ++--- a/broker/BUILD.bazel +++++ b/broker/BUILD.bazel ++@@ -44,6 +44,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:io_opentelemetry_opentelemetry_sdk", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++index 6af5afc14..39af18b9f 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++@@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongCounter; ++ import io.opentelemetry.api.metrics.LongHistogram; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.api.metrics.ObservableLongGauge; ++-import io.opentelemetry.exporter.logging.LoggingMetricExporter; +++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; ++ import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; ++@@ -36,6 +36,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; ++ import io.opentelemetry.sdk.metrics.View; ++ import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +++import io.opentelemetry.sdk.metrics.export.MetricExporter; ++ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; ++ import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; ++ import io.opentelemetry.sdk.resources.Resource; ++@@ -113,7 +114,7 @@ public class BrokerMetricsManager { ++ private OtlpGrpcMetricExporter metricExporter; ++ private PeriodicMetricReader periodicMetricReader; ++ private PrometheusHttpServer prometheusHttpServer; ++- private LoggingMetricExporter loggingMetricExporter; +++ private MetricExporter loggingMetricExporter; ++ private Meter brokerMeter; ++ ++ public static Supplier attributesBuilderSupplier = Attributes::builder; ++@@ -327,8 +328,8 @@ public class BrokerMetricsManager { ++ if (metricsExporterType == MetricsExporterType.LOG) { ++ SLF4JBridgeHandler.removeHandlersForRootLogger(); ++ SLF4JBridgeHandler.install(); ++- loggingMetricExporter = LoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); ++- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); +++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); +++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); ++ periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) ++ .setInterval(brokerConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) ++ .build(); ++diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml ++index 3c51e59d4..32dc29736 100644 ++--- a/broker/src/main/resources/rmq.broker.logback.xml +++++ b/broker/src/main/resources/rmq.broker.logback.xml ++@@ -672,6 +672,11 @@ ++ ++ ++ +++ +++ +++ +++ +++ ++ ++ ++ ++diff --git a/common/BUILD.bazel b/common/BUILD.bazel ++index a95a19ccd..e6701d0fc 100644 ++--- a/common/BUILD.bazel +++++ b/common/BUILD.bazel ++@@ -35,6 +35,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_sdk", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:org_apache_commons_commons_lang3", ++ "@maven//:org_lz4_lz4_java", ++ "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", ++diff --git a/common/pom.xml b/common/pom.xml ++index 31eb0f087..accc7f0a8 100644 ++--- a/common/pom.xml +++++ b/common/pom.xml ++@@ -80,6 +80,10 @@ ++ io.opentelemetry ++ opentelemetry-sdk ++ +++ +++ io.opentelemetry +++ opentelemetry-exporter-logging-otlp +++ ++ ++ io.grpc ++ grpc-stub ++diff --git a/controller/BUILD.bazel b/controller/BUILD.bazel ++index 843d9dc77..b2b743eb2 100644 ++--- a/controller/BUILD.bazel +++++ b/controller/BUILD.bazel ++@@ -49,6 +49,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:org_slf4j_jul_to_slf4j", ++ ], ++ ) ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++index 650740bcc..be9e77eea 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++@@ -26,7 +26,7 @@ import io.opentelemetry.api.metrics.LongHistogram; ++ import io.opentelemetry.api.metrics.LongUpDownCounter; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.api.metrics.ObservableLongGauge; ++-import io.opentelemetry.exporter.logging.LoggingMetricExporter; +++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; ++ import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; ++@@ -38,6 +38,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProvider; ++ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; ++ import io.opentelemetry.sdk.metrics.View; ++ import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +++import io.opentelemetry.sdk.metrics.export.MetricExporter; ++ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; ++ import io.opentelemetry.sdk.resources.Resource; ++ import java.io.File; ++@@ -121,7 +122,7 @@ public class ControllerMetricsManager { ++ ++ private PrometheusHttpServer prometheusHttpServer; ++ ++- private LoggingMetricExporter loggingMetricExporter; +++ private MetricExporter loggingMetricExporter; ++ ++ public static ControllerMetricsManager getInstance(ControllerManager controllerManager) { ++ if (instance == null) { ++@@ -364,8 +365,8 @@ public class ControllerMetricsManager { ++ if (type == MetricsExporterType.LOG) { ++ SLF4JBridgeHandler.removeHandlersForRootLogger(); ++ SLF4JBridgeHandler.install(); ++- loggingMetricExporter = LoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); ++- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); +++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); +++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); ++ periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) ++ .setInterval(config.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) ++ .build(); ++diff --git a/pom.xml b/pom.xml ++index 9f0b3eb96..4b382c6da 100644 ++--- a/pom.xml +++++ b/pom.xml ++@@ -974,6 +974,11 @@ ++ opentelemetry-sdk ++ ${opentelemetry.version} ++ +++ +++ io.opentelemetry +++ opentelemetry-exporter-logging-otlp +++ ${opentelemetry.version} +++ ++ ++ org.slf4j ++ jul-to-slf4j ++diff --git a/proxy/BUILD.bazel b/proxy/BUILD.bazel ++index b4f3c16e2..cb7af9254 100644 ++--- a/proxy/BUILD.bazel +++++ b/proxy/BUILD.bazel ++@@ -52,6 +52,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:io_opentelemetry_opentelemetry_sdk", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java ++index f5050858f..2b8dac5d8 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java ++@@ -21,15 +21,16 @@ import io.opentelemetry.api.common.Attributes; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.api.metrics.ObservableLongGauge; +++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; ++ import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; ++-import io.opentelemetry.exporter.logging.LoggingMetricExporter; ++ import io.opentelemetry.sdk.OpenTelemetrySdk; ++ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.SdkMeterProvider; ++ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; ++ import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +++import io.opentelemetry.sdk.metrics.export.MetricExporter; ++ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; ++ import io.opentelemetry.sdk.resources.Resource; ++ import java.util.HashMap; ++@@ -42,9 +43,9 @@ import org.apache.rocketmq.broker.metrics.BrokerMetricsManager; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.metrics.MetricsExporterType; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; ++-import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +++import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.slf4j.bridge.SLF4JBridgeHandler; ++ ++ import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.AGGREGATION_DELTA; ++@@ -67,7 +68,7 @@ public class ProxyMetricsManager implements StartAndShutdown { ++ private OtlpGrpcMetricExporter metricExporter; ++ private PeriodicMetricReader periodicMetricReader; ++ private PrometheusHttpServer prometheusHttpServer; ++- private LoggingMetricExporter loggingMetricExporter; +++ private MetricExporter loggingMetricExporter; ++ ++ public static ObservableLongGauge proxyUp = null; ++ ++@@ -221,8 +222,8 @@ public class ProxyMetricsManager implements StartAndShutdown { ++ if (metricsExporterType == MetricsExporterType.LOG) { ++ SLF4JBridgeHandler.removeHandlersForRootLogger(); ++ SLF4JBridgeHandler.install(); ++- loggingMetricExporter = LoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); ++- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); +++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); +++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); ++ periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) ++ .setInterval(proxyConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) ++ .build(); ++diff --git a/proxy/src/main/resources/rmq.proxy.logback.xml b/proxy/src/main/resources/rmq.proxy.logback.xml ++index d38827f92..f968a45e6 100644 ++--- a/proxy/src/main/resources/rmq.proxy.logback.xml +++++ b/proxy/src/main/resources/rmq.proxy.logback.xml ++@@ -418,6 +418,11 @@ ++ ++ ++ +++ +++ +++ +++ +++ ++ ++ ++ ++diff --git a/tieredstore/BUILD.bazel b/tieredstore/BUILD.bazel ++index 5b3885a4e..dea2c561b 100644 ++--- a/tieredstore/BUILD.bazel +++++ b/tieredstore/BUILD.bazel ++@@ -36,6 +36,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_sdk", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:org_apache_commons_commons_lang3", ++ "@maven//:org_apache_tomcat_annotations_api", ++ "@maven//:com_alibaba_fastjson", ++-- ++2.32.0.windows.2 ++ ++ ++From 1a681bdf9b5c5ab0be446d6394c0cac8768f45d9 Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Thu, 21 Sep 2023 19:58:29 +0800 ++Subject: [PATCH 03/12] [maven-release-plugin] prepare release ++ rocketmq-all-5.1.4 (#7377) ++ ++--- ++ acl/pom.xml | 2 +- ++ broker/pom.xml | 2 +- ++ client/pom.xml | 2 +- ++ common/pom.xml | 2 +- ++ container/pom.xml | 2 +- ++ controller/pom.xml | 2 +- ++ distribution/pom.xml | 2 +- ++ example/pom.xml | 2 +- ++ filter/pom.xml | 2 +- ++ namesrv/pom.xml | 2 +- ++ openmessaging/pom.xml | 2 +- ++ pom.xml | 4 ++-- ++ proxy/pom.xml | 2 +- ++ remoting/pom.xml | 2 +- ++ srvutil/pom.xml | 2 +- ++ store/pom.xml | 2 +- ++ test/pom.xml | 2 +- ++ tieredstore/pom.xml | 2 +- ++ tools/pom.xml | 2 +- ++ 19 files changed, 20 insertions(+), 20 deletions(-) ++ ++diff --git a/acl/pom.xml b/acl/pom.xml ++index 989c0cf77..9f6838b00 100644 ++--- a/acl/pom.xml +++++ b/acl/pom.xml ++@@ -13,7 +13,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ rocketmq-acl ++ rocketmq-acl ${project.version} ++diff --git a/broker/pom.xml b/broker/pom.xml ++index 16e026276..d483e67ba 100644 ++--- a/broker/pom.xml +++++ b/broker/pom.xml ++@@ -13,7 +13,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/client/pom.xml b/client/pom.xml ++index c59a43889..4febedc6d 100644 ++--- a/client/pom.xml +++++ b/client/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/common/pom.xml b/common/pom.xml ++index accc7f0a8..b70873dfa 100644 ++--- a/common/pom.xml +++++ b/common/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/container/pom.xml b/container/pom.xml ++index c8499f127..e6c1f4b4d 100644 ++--- a/container/pom.xml +++++ b/container/pom.xml ++@@ -18,7 +18,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/controller/pom.xml b/controller/pom.xml ++index 3346c7c82..46a3834c6 100644 ++--- a/controller/pom.xml +++++ b/controller/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ 4.0.0 ++ jar ++diff --git a/distribution/pom.xml b/distribution/pom.xml ++index dbde2d9d4..346c4de35 100644 ++--- a/distribution/pom.xml +++++ b/distribution/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ rocketmq-distribution ++ rocketmq-distribution ${project.version} ++diff --git a/example/pom.xml b/example/pom.xml ++index 862fc3169..9e7db43f8 100644 ++--- a/example/pom.xml +++++ b/example/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/filter/pom.xml b/filter/pom.xml ++index 3fe51ceae..84189066d 100644 ++--- a/filter/pom.xml +++++ b/filter/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/namesrv/pom.xml b/namesrv/pom.xml ++index 684b2683c..7c218078a 100644 ++--- a/namesrv/pom.xml +++++ b/namesrv/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml ++index aaa4c896c..fd499e3de 100644 ++--- a/openmessaging/pom.xml +++++ b/openmessaging/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/pom.xml b/pom.xml ++index 4b382c6da..0e1d04f15 100644 ++--- a/pom.xml +++++ b/pom.xml ++@@ -28,7 +28,7 @@ ++ 2012 ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ pom ++ Apache RocketMQ ${project.version} ++ http://rocketmq.apache.org/ ++@@ -37,7 +37,7 @@ ++ git@github.com:apache/rocketmq.git ++ scm:git:git@github.com:apache/rocketmq.git ++ scm:git:git@github.com:apache/rocketmq.git ++- HEAD +++ rocketmq-all-5.1.4 ++ ++ ++ ++diff --git a/proxy/pom.xml b/proxy/pom.xml ++index 3fbea107a..abf242eee 100644 ++--- a/proxy/pom.xml +++++ b/proxy/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/remoting/pom.xml b/remoting/pom.xml ++index 8a43c5c30..fc70cb62e 100644 ++--- a/remoting/pom.xml +++++ b/remoting/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/srvutil/pom.xml b/srvutil/pom.xml ++index fa54ad019..d7f946cce 100644 ++--- a/srvutil/pom.xml +++++ b/srvutil/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/store/pom.xml b/store/pom.xml ++index 38f04009d..6d6983c5d 100644 ++--- a/store/pom.xml +++++ b/store/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/test/pom.xml b/test/pom.xml ++index 8f25c35c9..39090e426 100644 ++--- a/test/pom.xml +++++ b/test/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml ++index c476040ba..7b209751f 100644 ++--- a/tieredstore/pom.xml +++++ b/tieredstore/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/tools/pom.xml b/tools/pom.xml ++index 1c3b431bc..806787ec9 100644 ++--- a/tools/pom.xml +++++ b/tools/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++-- ++2.32.0.windows.2 ++ ++ ++From 73b3fde83765e066541e3455cd1e6604292a9b7c Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Fri, 22 Sep 2023 10:08:59 +0800 ++Subject: [PATCH 04/12] [maven-release-plugin] prepare for next development ++ iteration (#7379) ++ ++--- ++ acl/pom.xml | 2 +- ++ broker/pom.xml | 2 +- ++ client/pom.xml | 2 +- ++ common/pom.xml | 2 +- ++ container/pom.xml | 2 +- ++ controller/pom.xml | 2 +- ++ distribution/pom.xml | 2 +- ++ example/pom.xml | 2 +- ++ filter/pom.xml | 2 +- ++ namesrv/pom.xml | 2 +- ++ openmessaging/pom.xml | 2 +- ++ pom.xml | 4 ++-- ++ proxy/pom.xml | 2 +- ++ remoting/pom.xml | 2 +- ++ srvutil/pom.xml | 2 +- ++ store/pom.xml | 2 +- ++ test/pom.xml | 2 +- ++ tieredstore/pom.xml | 2 +- ++ tools/pom.xml | 2 +- ++ 19 files changed, 20 insertions(+), 20 deletions(-) ++ ++diff --git a/acl/pom.xml b/acl/pom.xml ++index 9f6838b00..8a296e5ae 100644 ++--- a/acl/pom.xml +++++ b/acl/pom.xml ++@@ -13,7 +13,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ rocketmq-acl ++ rocketmq-acl ${project.version} ++diff --git a/broker/pom.xml b/broker/pom.xml ++index d483e67ba..add83045d 100644 ++--- a/broker/pom.xml +++++ b/broker/pom.xml ++@@ -13,7 +13,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/client/pom.xml b/client/pom.xml ++index 4febedc6d..d6fb3889b 100644 ++--- a/client/pom.xml +++++ b/client/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/common/pom.xml b/common/pom.xml ++index b70873dfa..6104c3ac6 100644 ++--- a/common/pom.xml +++++ b/common/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/container/pom.xml b/container/pom.xml ++index e6c1f4b4d..8af231e01 100644 ++--- a/container/pom.xml +++++ b/container/pom.xml ++@@ -18,7 +18,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/controller/pom.xml b/controller/pom.xml ++index 46a3834c6..8432b220b 100644 ++--- a/controller/pom.xml +++++ b/controller/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ 4.0.0 ++ jar ++diff --git a/distribution/pom.xml b/distribution/pom.xml ++index 346c4de35..73474d34a 100644 ++--- a/distribution/pom.xml +++++ b/distribution/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ rocketmq-distribution ++ rocketmq-distribution ${project.version} ++diff --git a/example/pom.xml b/example/pom.xml ++index 9e7db43f8..a8c7f5382 100644 ++--- a/example/pom.xml +++++ b/example/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/filter/pom.xml b/filter/pom.xml ++index 84189066d..892f46e9d 100644 ++--- a/filter/pom.xml +++++ b/filter/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/namesrv/pom.xml b/namesrv/pom.xml ++index 7c218078a..e320ed573 100644 ++--- a/namesrv/pom.xml +++++ b/namesrv/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml ++index fd499e3de..f10c8af6f 100644 ++--- a/openmessaging/pom.xml +++++ b/openmessaging/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/pom.xml b/pom.xml ++index 0e1d04f15..4202d4095 100644 ++--- a/pom.xml +++++ b/pom.xml ++@@ -28,7 +28,7 @@ ++ 2012 ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ pom ++ Apache RocketMQ ${project.version} ++ http://rocketmq.apache.org/ ++@@ -37,7 +37,7 @@ ++ git@github.com:apache/rocketmq.git ++ scm:git:git@github.com:apache/rocketmq.git ++ scm:git:git@github.com:apache/rocketmq.git ++- rocketmq-all-5.1.4 +++ HEAD ++ ++ ++ ++diff --git a/proxy/pom.xml b/proxy/pom.xml ++index abf242eee..5c5349a8c 100644 ++--- a/proxy/pom.xml +++++ b/proxy/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/remoting/pom.xml b/remoting/pom.xml ++index fc70cb62e..f78480680 100644 ++--- a/remoting/pom.xml +++++ b/remoting/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/srvutil/pom.xml b/srvutil/pom.xml ++index d7f946cce..894e9cc6f 100644 ++--- a/srvutil/pom.xml +++++ b/srvutil/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/store/pom.xml b/store/pom.xml ++index 6d6983c5d..e979030e8 100644 ++--- a/store/pom.xml +++++ b/store/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/test/pom.xml b/test/pom.xml ++index 39090e426..168cbab0b 100644 ++--- a/test/pom.xml +++++ b/test/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml ++index 7b209751f..b2ea40bf3 100644 ++--- a/tieredstore/pom.xml +++++ b/tieredstore/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/tools/pom.xml b/tools/pom.xml ++index 806787ec9..e1daa57a6 100644 ++--- a/tools/pom.xml +++++ b/tools/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++-- ++2.32.0.windows.2 ++ ++ ++From 88a9d939ce110381b3b418370d4711c0c214dc7f Mon Sep 17 00:00:00 2001 ++From: Ji Juntao ++Date: Sat, 23 Sep 2023 17:38:27 +0800 ++Subject: [PATCH 05/12] [ISSUE #7381] Fix the problem of inaccurate timer ++ message metric (#7382) ++ ++* correct the timerMetrics' result. ++ ++* for further extension. ++ ++* checkstyle. ++ ++* use toLong. ++--- ++ .../store/timer/TimerMessageStore.java | 20 +++++++++++++++---- ++ .../rocketmq/store/timer/TimerMetrics.java | 5 ++++- ++ .../rocketmq/store/timer/TimerRequest.java | 7 +++++-- ++ .../store/timer/TimerMetricsTest.java | 10 ++++++++-- ++ 4 files changed, 33 insertions(+), 9 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++index 0d50de65a..ac4c61cd6 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++@@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.function.Function; ++ import org.apache.commons.collections.CollectionUtils; +++import org.apache.commons.lang3.math.NumberUtils; ++ import org.apache.rocketmq.common.ServiceThread; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.TopicFilterType; ++@@ -599,7 +600,12 @@ public class TimerMessageStore { ++ if (null == msg || null == msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC)) { ++ return; ++ } ++- timerMetrics.addAndGet(msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC), value); +++ if (msg.getProperty(TIMER_ENQUEUE_MS) != null +++ && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) { +++ return; +++ } +++ // pass msg into addAndGet, for further more judgement extension. +++ timerMetrics.addAndGet(msg, value); ++ } catch (Throwable t) { ++ if (frequency.incrementAndGet() % 1000 == 0) { ++ LOGGER.error("error in adding metric", t); ++@@ -1323,6 +1329,7 @@ public class TimerMessageStore { ++ perfCounterTicks.startTick(ENQUEUE_PUT); ++ DefaultStoreMetricsManager.incTimerEnqueueCount(getRealTopic(req.getMsg())); ++ if (shouldRunningDequeue && req.getDelayTime() < currWriteTimeMs) { +++ req.setEnqueueTime(Long.MAX_VALUE); ++ dequeuePutQueue.put(req); ++ } else { ++ boolean doEnqueueRes = doEnqueue( ++@@ -1452,9 +1459,14 @@ public class TimerMessageStore { ++ } ++ try { ++ perfCounterTicks.startTick(DEQUEUE_PUT); ++- DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(tr.getMsg())); ++- addMetric(tr.getMsg(), -1); ++- MessageExtBrokerInner msg = convert(tr.getMsg(), tr.getEnqueueTime(), needRoll(tr.getMagic())); +++ MessageExt msgExt = tr.getMsg(); +++ DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(msgExt)); +++ if (tr.getEnqueueTime() == Long.MAX_VALUE) { +++ // never enqueue, mark it. +++ MessageAccessor.putProperty(msgExt, TIMER_ENQUEUE_MS, String.valueOf(Long.MAX_VALUE)); +++ } +++ addMetric(msgExt, -1); +++ MessageExtBrokerInner msg = convert(msgExt, tr.getEnqueueTime(), needRoll(tr.getMagic())); ++ doRes = PUT_NEED_RETRY != doPut(msg, needRoll(tr.getMagic())); ++ while (!doRes && !isStopped()) { ++ if (!isRunningDequeue()) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java ++index e7b00cc07..7f8fedd8a 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java ++@@ -38,6 +38,8 @@ import java.util.concurrent.locks.Lock; ++ import java.util.concurrent.locks.ReentrantLock; ++ import org.apache.rocketmq.common.ConfigManager; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.message.MessageConst; +++import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++@@ -78,7 +80,8 @@ public class TimerMetrics extends ConfigManager { ++ return distPair.getCount().addAndGet(value); ++ } ++ ++- public long addAndGet(String topic, int value) { +++ public long addAndGet(MessageExt msg, int value) { +++ String topic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC); ++ Metric pair = getTopicPair(topic); ++ getDataVersion().nextVersion(); ++ pair.setTimeStamp(System.currentTimeMillis()); ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java ++index 1dd64f759..1b25d355c 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java ++@@ -27,8 +27,9 @@ public class TimerRequest { ++ private final int sizePy; ++ private final long delayTime; ++ ++- private final long enqueueTime; ++ private final int magic; +++ +++ private long enqueueTime; ++ private MessageExt msg; ++ ++ ++@@ -94,7 +95,9 @@ public class TimerRequest { ++ public void setLatch(CountDownLatch latch) { ++ this.latch = latch; ++ } ++- +++ public void setEnqueueTime(long enqueueTime) { +++ this.enqueueTime = enqueueTime; +++ } ++ public void idempotentRelease() { ++ idempotentRelease(true); ++ } ++diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java ++index b7392cc45..3c7b9b67f 100644 ++--- a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java +++++ b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java ++@@ -16,6 +16,9 @@ ++ */ ++ package org.apache.rocketmq.store.timer; ++ +++import org.apache.rocketmq.common.message.MessageAccessor; +++import org.apache.rocketmq.common.message.MessageConst; +++import org.apache.rocketmq.common.message.MessageExt; ++ import org.junit.Assert; ++ import org.junit.Test; ++ ++@@ -31,8 +34,11 @@ public class TimerMetricsTest { ++ ++ TimerMetrics first = new TimerMetrics(baseDir); ++ Assert.assertTrue(first.load()); ++- first.addAndGet("AAA", 1000); ++- first.addAndGet("BBB", 2000); +++ MessageExt msg = new MessageExt(); +++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "AAA"); +++ first.addAndGet(msg, 1000); +++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "BBB"); +++ first.addAndGet(msg, 2000); ++ Assert.assertEquals(1000, first.getTimingCount("AAA")); ++ Assert.assertEquals(2000, first.getTimingCount("BBB")); ++ long curr = System.currentTimeMillis(); ++-- ++2.32.0.windows.2 ++ ++ ++From d7e5c4d1a4e048cd97f0b29a96a0fc575927a03e Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Mon, 25 Sep 2023 13:37:36 +0800 ++Subject: [PATCH 06/12] [ISSUE #7389] Fix the problem that getLastMappedFile ++ function affects performance ++ ++Co-authored-by: guyinyou ++--- ++ .../apache/rocketmq/store/MappedFileQueue.java | 15 +++++++++++++-- ++ 1 file changed, 13 insertions(+), 2 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++index 32b90d14f..9a0824829 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++@@ -377,8 +377,19 @@ public class MappedFileQueue implements Swappable { ++ } ++ ++ public MappedFile getLastMappedFile() { ++- MappedFile[] mappedFiles = this.mappedFiles.toArray(new MappedFile[0]); ++- return mappedFiles.length == 0 ? null : mappedFiles[mappedFiles.length - 1]; +++ MappedFile mappedFileLast = null; +++ while (!this.mappedFiles.isEmpty()) { +++ try { +++ mappedFileLast = this.mappedFiles.get(this.mappedFiles.size() - 1); +++ break; +++ } catch (IndexOutOfBoundsException e) { +++ //continue; +++ } catch (Exception e) { +++ log.error("getLastMappedFile has exception.", e); +++ break; +++ } +++ } +++ return mappedFileLast; ++ } ++ ++ public boolean resetOffset(long offset) { ++-- ++2.32.0.windows.2 ++ ++ ++From 3fd43353fdf880deb5d63ba3ad50cc6e3259dc3a Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Tue, 26 Sep 2023 13:53:51 +0800 ++Subject: [PATCH 07/12] [ISSUE #7393] Add timeout configuration for grpc server ++ (#7394) ++ ++* Add timeout configuration for grpc server ++ ++* Add proxyConfig ++--- ++ .../java/org/apache/rocketmq/proxy/ProxyStartup.java | 1 + ++ .../apache/rocketmq/proxy/config/ProxyConfig.java | 9 +++++++++ ++ .../org/apache/rocketmq/proxy/grpc/GrpcServer.java | 10 ++++++++-- ++ .../rocketmq/proxy/grpc/GrpcServerBuilder.java | 12 +++++++++++- ++ 4 files changed, 29 insertions(+), 3 deletions(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java ++index 06d5f4525..3b2ca99bf 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java ++@@ -85,6 +85,7 @@ public class ProxyStartup { ++ .addService(ChannelzService.newInstance(100)) ++ .addService(ProtoReflectionService.newInstance()) ++ .configInterceptor(accessValidators) +++ .shutdownTime(ConfigurationManager.getProxyConfig().getGrpcShutdownTimeSeconds(), TimeUnit.SECONDS) ++ .build(); ++ PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer); ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index b2478fec3..c0d00d864 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -87,6 +87,7 @@ public class ProxyConfig implements ConfigFile { ++ */ ++ private String proxyMode = ProxyMode.CLUSTER.name(); ++ private Integer grpcServerPort = 8081; +++ private long grpcShutdownTimeSeconds = 30; ++ private int grpcBossLoopNum = 1; ++ private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2; ++ private boolean enableGrpcEpoll = false; ++@@ -443,6 +444,14 @@ public class ProxyConfig implements ConfigFile { ++ this.grpcServerPort = grpcServerPort; ++ } ++ +++ public long getGrpcShutdownTimeSeconds() { +++ return grpcShutdownTimeSeconds; +++ } +++ +++ public void setGrpcShutdownTimeSeconds(long grpcShutdownTimeSeconds) { +++ this.grpcShutdownTimeSeconds = grpcShutdownTimeSeconds; +++ } +++ ++ public boolean isUseEndpointPortFromRequest() { ++ return useEndpointPortFromRequest; ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java ++index 1bffa3c0b..d5b896fe1 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java ++@@ -29,8 +29,14 @@ public class GrpcServer implements StartAndShutdown { ++ ++ private final Server server; ++ ++- protected GrpcServer(Server server) { +++ private final long timeout; +++ +++ private final TimeUnit unit; +++ +++ protected GrpcServer(Server server, long timeout, TimeUnit unit) { ++ this.server = server; +++ this.timeout = timeout; +++ this.unit = unit; ++ } ++ ++ public void start() throws Exception { ++@@ -40,7 +46,7 @@ public class GrpcServer implements StartAndShutdown { ++ ++ public void shutdown() { ++ try { ++- this.server.shutdown().awaitTermination(30, TimeUnit.SECONDS); +++ this.server.shutdown().awaitTermination(timeout, unit); ++ log.info("grpc server shutdown successfully."); ++ } catch (Exception e) { ++ e.printStackTrace(); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java ++index 9cddd3013..0e79006f6 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java ++@@ -41,6 +41,10 @@ public class GrpcServerBuilder { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); ++ protected NettyServerBuilder serverBuilder; ++ +++ protected long time = 30; +++ +++ protected TimeUnit unit = TimeUnit.SECONDS; +++ ++ public static GrpcServerBuilder newBuilder(ThreadPoolExecutor executor, int port) { ++ return new GrpcServerBuilder(executor, port); ++ } ++@@ -77,6 +81,12 @@ public class GrpcServerBuilder { ++ port, bossLoopNum, workerLoopNum, maxInboundMessageSize); ++ } ++ +++ public GrpcServerBuilder shutdownTime(long time, TimeUnit unit) { +++ this.time = time; +++ this.unit = unit; +++ return this; +++ } +++ ++ public GrpcServerBuilder addService(BindableService service) { ++ this.serverBuilder.addService(service); ++ return this; ++@@ -93,7 +103,7 @@ public class GrpcServerBuilder { ++ } ++ ++ public GrpcServer build() { ++- return new GrpcServer(this.serverBuilder.build()); +++ return new GrpcServer(this.serverBuilder.build(), time, unit); ++ } ++ ++ public GrpcServerBuilder configInterceptor(List accessValidators) { ++-- ++2.32.0.windows.2 ++ ++ ++From c3b86cd1e3c068bc5847671c899a284e49a2ecdc Mon Sep 17 00:00:00 2001 ++From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= ++ ++Date: Tue, 26 Sep 2023 16:07:13 +0800 ++Subject: [PATCH 08/12] [ISSUE #7398] Fix ExportConfigsCommand NPE (#7399) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Co-authored-by: 石臻臻 ++--- ++ .../command/export/ExportConfigsCommand.java | 42 ++++++++++++------- ++ 1 file changed, 26 insertions(+), 16 deletions(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++index 03613b29c..c3f96d597 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++@@ -20,6 +20,7 @@ import java.util.HashMap; ++ import java.util.List; ++ import java.util.Map; ++ import java.util.Map.Entry; +++import java.util.Arrays; ++ import java.util.Properties; ++ ++ import com.alibaba.fastjson.JSON; ++@@ -106,24 +107,33 @@ public class ExportConfigsCommand implements SubCommand { ++ } ++ } ++ +++ ++ private Properties needBrokerProprties(Properties properties) { +++ List propertyKeys = Arrays.asList( +++ "brokerClusterName", +++ "brokerId", +++ "brokerName", +++ "brokerRole", +++ "fileReservedTime", +++ "filterServerNums", +++ "flushDiskType", +++ "maxMessageSize", +++ "messageDelayLevel", +++ "msgTraceTopicName", +++ "slaveReadEnable", +++ "traceOn", +++ "traceTopicEnable", +++ "useTLS", +++ "autoCreateTopicEnable", +++ "autoCreateSubscriptionGroup" +++ ); +++ ++ Properties newProperties = new Properties(); ++- newProperties.setProperty("brokerClusterName", properties.getProperty("brokerClusterName")); ++- newProperties.setProperty("brokerId", properties.getProperty("brokerId")); ++- newProperties.setProperty("brokerName", properties.getProperty("brokerName")); ++- newProperties.setProperty("brokerRole", properties.getProperty("brokerRole")); ++- newProperties.setProperty("fileReservedTime", properties.getProperty("fileReservedTime")); ++- newProperties.setProperty("filterServerNums", properties.getProperty("filterServerNums")); ++- newProperties.setProperty("flushDiskType", properties.getProperty("flushDiskType")); ++- newProperties.setProperty("maxMessageSize", properties.getProperty("maxMessageSize")); ++- newProperties.setProperty("messageDelayLevel", properties.getProperty("messageDelayLevel")); ++- newProperties.setProperty("msgTraceTopicName", properties.getProperty("msgTraceTopicName")); ++- newProperties.setProperty("slaveReadEnable", properties.getProperty("slaveReadEnable")); ++- newProperties.setProperty("traceOn", properties.getProperty("traceOn")); ++- newProperties.setProperty("traceTopicEnable", properties.getProperty("traceTopicEnable")); ++- newProperties.setProperty("useTLS", properties.getProperty("useTLS")); ++- newProperties.setProperty("autoCreateTopicEnable", properties.getProperty("autoCreateTopicEnable")); ++- newProperties.setProperty("autoCreateSubscriptionGroup", properties.getProperty("autoCreateSubscriptionGroup")); +++ propertyKeys.stream() +++ .filter(key -> properties.getProperty(key) != null) +++ .forEach(key -> newProperties.setProperty(key, properties.getProperty(key))); +++ ++ return newProperties; ++ } +++ ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 959a98120cea8022498557a308aff35e3d8def2a Mon Sep 17 00:00:00 2001 ++From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= ++ ++Date: Wed, 27 Sep 2023 01:59:58 +0800 ++Subject: [PATCH 09/12] [ISSUE #7400] Fix getBrokerEpochSubCommand NPE ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Co-authored-by: 石臻臻 ++--- ++ .../broker/processor/AdminBrokerProcessor.java | 10 ++++++++-- ++ 1 file changed, 8 insertions(+), 2 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++index 9e48431be..e77120e15 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++@@ -2736,10 +2736,16 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ final ReplicasManager replicasManager = this.brokerController.getReplicasManager(); ++ assert replicasManager != null; ++ final BrokerConfig brokerConfig = this.brokerController.getBrokerConfig(); +++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ +++ if (!brokerConfig.isEnableControllerMode()) { +++ response.setCode(ResponseCode.SYSTEM_ERROR); +++ response.setRemark("this request only for controllerMode "); +++ return response; +++ } ++ final EpochEntryCache entryCache = new EpochEntryCache(brokerConfig.getBrokerClusterName(), ++- brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset()); +++ brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset()); ++ ++- final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setBody(entryCache.encode()); ++ response.setCode(ResponseCode.SUCCESS); ++ response.setRemark(null); ++-- ++2.32.0.windows.2 ++ ++ ++From 0a6ae4605fef4eaab6262fbd370aba887d8ae58b Mon Sep 17 00:00:00 2001 ++From: tiger lee <1026203200@qq.com> ++Date: Wed, 27 Sep 2023 14:43:15 +0800 ++Subject: [PATCH 10/12] [ISSUE #7396] Fix wrong word in ++ BrokerController#doResterBrokerAll (#7397) ++ ++--- ++ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 53e2e1b62..d4bded600 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1807,7 +1807,7 @@ public class BrokerController { ++ TopicConfigSerializeWrapper topicConfigWrapper) { ++ ++ if (shutdown) { ++- BrokerController.LOG.info("BrokerController#doResterBrokerAll: broker has shutdown, no need to register any more."); +++ BrokerController.LOG.info("BrokerController#doRegisterBrokerAll: broker has shutdown, no need to register any more."); ++ return; ++ } ++ List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( ++-- ++2.32.0.windows.2 ++ ++ ++From 4f1b42a7c5557bcadd6b9982a0c9bd876622c69e Mon Sep 17 00:00:00 2001 ++From: ShuangxiDing ++Date: Thu, 28 Sep 2023 16:52:02 +0800 ++Subject: [PATCH 11/12] [ISSUE #7410] Handle the Exception when the Proxy ++ requests the client ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Co-authored-by: 徒钟 ++--- ++ .../remoting/channel/RemotingChannel.java | 23 ++++++++++++++----- ++ 1 file changed, 17 insertions(+), 6 deletions(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java ++index 40946cabf..d755fdcc4 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java ++@@ -34,6 +34,7 @@ import org.apache.rocketmq.common.utils.NetworkUtil; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.proxy.common.channel.ChannelHelper; +++import org.apache.rocketmq.proxy.common.utils.ExceptionUtils; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter; ++@@ -158,10 +159,15 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver ++ if (response.getCode() == ResponseCode.SUCCESS) { ++ ConsumerRunningInfo consumerRunningInfo = ConsumerRunningInfo.decode(response.getBody(), ConsumerRunningInfo.class); ++ responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", consumerRunningInfo)); +++ } else { +++ String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark()); +++ RuntimeException e = new RuntimeException(errMsg); +++ responseFuture.completeExceptionally(e); ++ } ++- String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark()); ++- RuntimeException e = new RuntimeException(errMsg); ++- responseFuture.completeExceptionally(e); +++ }) +++ .exceptionally(t -> { +++ responseFuture.completeExceptionally(ExceptionUtils.getRealException(t)); +++ return null; ++ }); ++ return CompletableFuture.completedFuture(null); ++ } catch (Throwable t) { ++@@ -183,10 +189,15 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver ++ if (response.getCode() == ResponseCode.SUCCESS) { ++ ConsumeMessageDirectlyResult result = ConsumeMessageDirectlyResult.decode(response.getBody(), ConsumeMessageDirectlyResult.class); ++ responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", result)); +++ } else { +++ String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark()); +++ RuntimeException e = new RuntimeException(errMsg); +++ responseFuture.completeExceptionally(e); ++ } ++- String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark()); ++- RuntimeException e = new RuntimeException(errMsg); ++- responseFuture.completeExceptionally(e); +++ }) +++ .exceptionally(t -> { +++ responseFuture.completeExceptionally(ExceptionUtils.getRealException(t)); +++ return null; ++ }); ++ return CompletableFuture.completedFuture(null); ++ } catch (Throwable t) { ++-- ++2.32.0.windows.2 ++ ++ ++From c36bb78e850129b9db40adc5b0e1b9bfd5c8fd2e Mon Sep 17 00:00:00 2001 ++From: shriVATSA54 <116296557+shriVATSA54@users.noreply.github.com> ++Date: Sat, 7 Oct 2023 12:22:39 +0530 ++Subject: [PATCH 12/12] [ISSUE 7313] Enhancement Optimization Method name ++ (#7420) ++ ++* Enhancment/method_name/#7313/ ++ ++* Enhancment/method_name/#7313/ ++ ++* Enhancment/method_name/#7313/ ++--- ++ .../tieredstore/provider/TieredStoreTopicBlackListFilter.java | 2 +- ++ .../rocketmq/tieredstore/provider/TieredStoreTopicFilter.java | 2 +- ++ .../provider/TieredStoreTopicBlackListFilterTest.java | 2 +- ++ 3 files changed, 3 insertions(+), 3 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++index 50adbb713..f8bf165bc 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++@@ -39,7 +39,7 @@ public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter { ++ } ++ ++ @Override ++- public void addTopicToWhiteList(String topicName) { +++ public void addTopicToBlackList(String topicName) { ++ this.topicBlackSet.add(topicName); ++ } ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++index 3f26b8b02..f983ed6e9 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++@@ -21,5 +21,5 @@ public interface TieredStoreTopicFilter { ++ ++ boolean filterTopic(String topicName); ++ ++- void addTopicToWhiteList(String topicName); +++ void addTopicToBlackList(String topicName); ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++index 2bf48173c..fbaafa1b4 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++@@ -30,7 +30,7 @@ public class TieredStoreTopicBlackListFilterTest { ++ ++ String topicName = "WhiteTopic"; ++ Assert.assertFalse(topicFilter.filterTopic(topicName)); ++- topicFilter.addTopicToWhiteList(topicName); +++ topicFilter.addTopicToBlackList(topicName); ++ Assert.assertTrue(topicFilter.filterTopic(topicName)); ++ } ++ } ++\ No newline at end of file ++-- ++2.32.0.windows.2 ++ +diff --git a/patch020-backport-add-goaway-mechanism.patch b/patch020-backport-add-goaway-mechanism.patch +new file mode 100644 +index 000000000..122db0325 +--- /dev/null ++++ b/patch020-backport-add-goaway-mechanism.patch +@@ -0,0 +1,3696 @@ ++From 84156084a4c5228e1d2fe21e068fff330bbc40d1 Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Sun, 8 Oct 2023 11:13:25 +0800 ++Subject: [PATCH 1/7] [ISSUE #7321] Refector NettyRemotingAbstract with unify ++ future implementation (#7322) ++ ++* Refector NettyRemotingAbstract ++ ++* Add invoke with future method ++ ++* Deprecate InvokeCallback#operationComplete ++ ++* Add operationSuccess and operationException for InvokeCallback ++ ++* fix unit test ++ ++* fix unit test ++ ++* Keep InvokeCallback#operationComplete ++ ++* Optimize invokeAsyncImpl operationComplete ++ ++* Add unit test for NettyRemotingClient ++ ++* fix checkstyle ++--- ++ .../rocketmq/broker/out/BrokerOuterAPI.java | 147 +++++---- ++ .../rocketmq/client/impl/MQAdminImpl.java | 71 ++-- ++ .../rocketmq/client/impl/MQClientAPIImpl.java | 239 +++++++------- ++ .../client/impl/mqclient/MQClientAPIExt.java | 309 ++++++++---------- ++ .../client/impl/MQClientAPIImplTest.java | 12 +- ++ .../remoting/RemotingProtocolServer.java | 22 +- ++ .../service/mqclient/MQClientAPIExtTest.java | 97 +++--- ++ .../rocketmq/remoting/InvokeCallback.java | 15 + ++ .../rocketmq/remoting/RemotingClient.java | 27 +- ++ .../remoting/netty/NettyRemotingAbstract.java | 123 ++++--- ++ .../remoting/netty/NettyRemotingClient.java | 33 +- ++ .../remoting/netty/ResponseFuture.java | 15 + ++ .../rocketmq/remoting/rpc/RpcClientImpl.java | 29 +- ++ .../rocketmq/remoting/RemotingServerTest.java | 22 +- ++ .../rocketmq/remoting/netty/MockChannel.java | 21 +- ++ .../remoting/netty/MockChannelPromise.java | 191 +++++++++++ ++ .../netty/NettyRemotingAbstractTest.java | 54 ++- ++ .../netty/NettyRemotingClientTest.java | 185 ++++++++++- ++ 18 files changed, 1029 insertions(+), 583 deletions(-) ++ rename client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java => remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java (57%) ++ create mode 100644 remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++index 9dfb8127d..6fde48dd9 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++@@ -73,6 +73,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.netty.NettyClientConfig; ++ import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +++import org.apache.rocketmq.remoting.netty.ResponseFuture; ++ import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo; ++ import org.apache.rocketmq.remoting.protocol.DataVersion; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -107,6 +108,8 @@ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; ++ import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader; ++@@ -124,8 +127,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerReques ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterTopicRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; ++ import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult; ++ import org.apache.rocketmq.remoting.protocol.route.BrokerData; ++@@ -151,7 +152,6 @@ public class BrokerOuterAPI { ++ private final RpcClient rpcClient; ++ private String nameSrvAddr = null; ++ ++- ++ public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { ++ this(nettyClientConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata()); ++ } ++@@ -459,7 +459,7 @@ public class BrokerOuterAPI { ++ * @param filterServerList ++ * @param oneway ++ * @param timeoutMills ++- * @param compressed default false +++ * @param compressed default false ++ * @return ++ */ ++ public List registerBrokerAll( ++@@ -643,7 +643,6 @@ public class BrokerOuterAPI { ++ queueDatas.add(queueData); ++ final byte[] topicRouteBody = topicRouteData.encode(); ++ ++- ++ List nameServerAddressList = this.remotingClient.getNameServerAddressList(); ++ final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size()); ++ for (final String namesrvAddr : nameServerAddressList) { ++@@ -910,25 +909,33 @@ public class BrokerOuterAPI { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); ++ ++ request.setBody(requestBody.encode()); ++- this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { ++- if (callback == null) { ++- return; +++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ ++ } ++ ++- try { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), ++- LockBatchResponseBody.class); ++- Set messageQueues = responseBody.getLockOKMQSet(); ++- callback.onSuccess(messageQueues); ++- } else { ++- callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); ++- } +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ if (callback == null) { +++ return; ++ } ++- } catch (Throwable ignored) { +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), +++ LockBatchResponseBody.class); +++ Set messageQueues = responseBody.getLockOKMQSet(); +++ callback.onSuccess(messageQueues); +++ } else { +++ callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ } ++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ if (callback == null) { +++ return; +++ } +++ callback.onException(throwable); ++ } ++ }); ++ } ++@@ -942,22 +949,30 @@ public class BrokerOuterAPI { ++ ++ request.setBody(requestBody.encode()); ++ ++- this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { ++- if (callback == null) { ++- return; +++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ ++ } ++ ++- try { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- callback.onSuccess(); ++- } else { ++- callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); ++- } +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ if (callback == null) { +++ return; ++ } ++- } catch (Throwable ignored) { +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ callback.onSuccess(); +++ } else { +++ callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ } ++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ if (callback == null) { +++ return; +++ } +++ callback.onException(throwable); ++ } ++ }); ++ } ++@@ -983,21 +998,27 @@ public class BrokerOuterAPI { ++ CompletableFuture cf = new CompletableFuture<>(); ++ final String msgId = msg.getMsgId(); ++ try { ++- this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (null != response) { ++- SendResult sendResult = null; +++ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ try { ++- sendResult = this.processSendResponse(brokerName, msg, response); +++ SendResult sendResult = processSendResponse(brokerName, msg, response); ++ cf.complete(sendResult); ++ } catch (MQBrokerException | RemotingCommandException e) { ++ LOGGER.error("processSendResponse in sendMessageToSpecificBrokerAsync failed, msgId=" + msgId, e); ++ cf.completeExceptionally(e); ++ } ++- } else { ++- cf.complete(null); ++ } ++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ cf.completeExceptionally(throwable); +++ } ++ }); ++ } catch (Throwable t) { ++ LOGGER.error("invokeAsync failed in sendMessageToSpecificBrokerAsync, msgId=" + msgId, t); ++@@ -1057,7 +1078,7 @@ public class BrokerOuterAPI { ++ } ++ if (sendStatus != null) { ++ SendMessageResponseHeader responseHeader = ++- (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); +++ (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); ++ ++ //If namespace not null , reset Topic without namespace. ++ String topic = msg.getTopic(); ++@@ -1073,8 +1094,8 @@ public class BrokerOuterAPI { ++ uniqMsgId = sb.toString(); ++ } ++ SendResult sendResult = new SendResult(sendStatus, ++- uniqMsgId, ++- responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); +++ uniqMsgId, +++ responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); ++ sendResult.setTransactionId(responseHeader.getTransactionId()); ++ String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); ++ String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH); ++@@ -1218,8 +1239,9 @@ public class BrokerOuterAPI { ++ /** ++ * Broker try to elect itself as a master in broker set ++ */ ++- public Pair> brokerElect(String controllerAddress, String clusterName, String brokerName, ++- Long brokerId) throws Exception { +++ public Pair> brokerElect(String controllerAddress, String clusterName, +++ String brokerName, +++ Long brokerId) throws Exception { ++ ++ final ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, requestHeader); ++@@ -1237,7 +1259,8 @@ public class BrokerOuterAPI { ++ throw new MQBrokerException(response.getCode(), response.getRemark()); ++ } ++ ++- public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, final String controllerAddress) throws Exception { +++ public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, +++ final String controllerAddress) throws Exception { ++ final GetNextBrokerIdRequestHeader requestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName); ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, requestHeader); ++ final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); ++@@ -1248,7 +1271,8 @@ public class BrokerOuterAPI { ++ throw new MQBrokerException(response.getCode(), response.getRemark()); ++ } ++ ++- public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { +++ public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, +++ final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { ++ final ApplyBrokerIdRequestHeader requestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, brokerId, registerCheckCode); ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, requestHeader); ++ final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); ++@@ -1259,7 +1283,9 @@ public class BrokerOuterAPI { ++ throw new MQBrokerException(response.getCode(), response.getRemark()); ++ } ++ ++- public Pair> registerBrokerToController(final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, final String controllerAddress) throws Exception { +++ public Pair> registerBrokerToController( +++ final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, +++ final String controllerAddress) throws Exception { ++ final RegisterBrokerToControllerRequestHeader requestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, brokerId, brokerAddress); ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, requestHeader); ++ final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); ++@@ -1355,16 +1381,25 @@ public class BrokerOuterAPI { ++ ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader); ++ CompletableFuture pullResultFuture = new CompletableFuture<>(); ++- this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- if (responseFuture.getCause() != null) { ++- pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); ++- return; +++ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ ++ } ++- try { ++- PullResultExt pullResultExt = this.processPullResponse(responseFuture.getResponseCommand(), brokerAddr); ++- this.processPullResult(pullResultExt, brokerName, queueId); ++- pullResultFuture.complete(pullResultExt); ++- } catch (Exception e) { +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ try { +++ PullResultExt pullResultExt = processPullResponse(response, brokerAddr); +++ processPullResult(pullResultExt, brokerName, queueId); +++ pullResultFuture.complete(pullResultExt); +++ } catch (Exception e) { +++ pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); +++ } +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { ++ pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); ++ } ++ }); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java ++index 1ef3a9483..83835bd3d 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java ++@@ -44,6 +44,8 @@ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageId; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.common.utils.NetworkUtil; +++import org.apache.rocketmq.logging.org.slf4j.Logger; +++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.exception.RemotingCommandException; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++@@ -55,8 +57,6 @@ import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.route.BrokerData; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++-import org.apache.rocketmq.logging.org.slf4j.Logger; ++-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ public class MQAdminImpl { ++ ++@@ -357,44 +357,51 @@ public class MQAdminImpl { ++ new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ try { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- switch (response.getCode()) { ++- case ResponseCode.SUCCESS: { ++- QueryMessageResponseHeader responseHeader = null; ++- try { ++- responseHeader = ++- (QueryMessageResponseHeader) response ++- .decodeCommandCustomHeader(QueryMessageResponseHeader.class); ++- } catch (RemotingCommandException e) { ++- log.error("decodeCommandCustomHeader exception", e); ++- return; ++- } ++- ++- List wrappers = ++- MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); ++- ++- QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); ++- try { ++- lock.writeLock().lock(); ++- queryResultList.add(qr); ++- } finally { ++- lock.writeLock().unlock(); ++- } ++- break; +++ switch (response.getCode()) { +++ case ResponseCode.SUCCESS: { +++ QueryMessageResponseHeader responseHeader = null; +++ try { +++ responseHeader = +++ (QueryMessageResponseHeader) response +++ .decodeCommandCustomHeader(QueryMessageResponseHeader.class); +++ } catch (RemotingCommandException e) { +++ log.error("decodeCommandCustomHeader exception", e); +++ return; ++ } ++- default: ++- log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); ++- break; +++ +++ List wrappers = +++ MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); +++ +++ QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); +++ try { +++ lock.writeLock().lock(); +++ queryResultList.add(qr); +++ } finally { +++ lock.writeLock().unlock(); +++ } +++ break; ++ } ++- } else { ++- log.warn("getResponseCommand return null"); +++ default: +++ log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); +++ break; ++ } +++ ++ } finally { ++ countDownLatch.countDown(); ++ } ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ log.error("queryMessage error, requestHeader={}", requestHeader); +++ countDownLatch.countDown(); +++ } ++ }, isUniqKey); ++ } catch (Exception e) { ++ log.warn("queryMessage exception", e); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 3201a493f..2407e5737 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.Validators; ++-import org.apache.rocketmq.client.common.ClientErrorCode; ++ import org.apache.rocketmq.client.consumer.AckCallback; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.AckStatus; ++@@ -653,10 +652,13 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++- long cost = System.currentTimeMillis() - beginStartTime; ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (null == sendCallback && response != null) { ++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ long cost = System.currentTimeMillis() - beginStartTime; +++ if (null == sendCallback) { ++ try { ++ SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); ++ if (context != null && sendResult != null) { ++@@ -666,46 +668,47 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ } catch (Throwable e) { ++ } ++ ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); ++ return; ++ } ++ ++- if (response != null) { +++ try { +++ SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); +++ assert sendResult != null; +++ if (context != null) { +++ context.setSendResult(sendResult); +++ context.getProducer().executeSendMessageHookAfter(context); +++ } +++ ++ try { ++- SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); ++- assert sendResult != null; ++- if (context != null) { ++- context.setSendResult(sendResult); ++- context.getProducer().executeSendMessageHookAfter(context); ++- } +++ sendCallback.onSuccess(sendResult); +++ } catch (Throwable e) { +++ } ++ ++- try { ++- sendCallback.onSuccess(sendResult); ++- } catch (Throwable e) { ++- } +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); +++ } catch (Exception e) { +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); +++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +++ retryTimesWhenSendFailed, times, e, context, false, producer); +++ } +++ } ++ ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); ++- } catch (Exception e) { ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); ++- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++- retryTimesWhenSendFailed, times, e, context, false, producer); ++- } +++ @Override +++ public void operationFail(Throwable throwable) { +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); +++ long cost = System.currentTimeMillis() - beginStartTime; +++ if (throwable instanceof RemotingSendRequestException) { +++ MQClientException ex = new MQClientException("send request failed", throwable); +++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +++ retryTimesWhenSendFailed, times, ex, context, true, producer); +++ } else if (throwable instanceof RemotingTimeoutException) { +++ MQClientException ex = new MQClientException("wait response timeout, cost=" + cost, throwable); +++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +++ retryTimesWhenSendFailed, times, ex, context, true, producer); ++ } else { ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); ++- if (!responseFuture.isSendRequestOK()) { ++- MQClientException ex = new MQClientException("send request failed", responseFuture.getCause()); ++- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++- retryTimesWhenSendFailed, times, ex, context, true, producer); ++- } else if (responseFuture.isTimeout()) { ++- MQClientException ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", ++- responseFuture.getCause()); ++- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++- retryTimesWhenSendFailed, times, ex, context, true, producer); ++- } else { ++- MQClientException ex = new MQClientException("unknow reseaon", responseFuture.getCause()); ++- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++- retryTimesWhenSendFailed, times, ex, context, true, producer); ++- } +++ MQClientException ex = new MQClientException("unknow reseaon", throwable); +++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +++ retryTimesWhenSendFailed, times, ex, context, true, producer); ++ } ++ } ++ }); ++@@ -857,30 +860,25 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ final long timeoutMillis, final PopCallback popCallback ++ ) throws RemotingException, InterruptedException { ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader); ++- this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { +++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++- public void onComplete(ResponseFuture responseFuture) { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- PopResult ++- popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); ++- assert popResult != null; ++- popCallback.onSuccess(popResult); ++- } catch (Exception e) { ++- popCallback.onException(e); ++- } ++- } else { ++- if (!responseFuture.isSendRequestOK()) { ++- popCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- popCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, ++- responseFuture.getCause())); ++- } else { ++- popCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); ++- } +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ try { +++ PopResult popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); +++ popCallback.onSuccess(popResult); +++ } catch (Exception e) { +++ popCallback.onException(e); ++ } ++ } +++ @Override +++ public void operationFail(Throwable throwable) { +++ popCallback.onException(throwable); +++ } ++ }); ++ } ++ ++@@ -959,34 +957,26 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ request.setBody(requestBody.encode()); ++ } ++ } ++- this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) { +++ this.remotingClient.invokeAsync(addr, request, timeOut, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } ++ ++ @Override ++- public void onComplete(ResponseFuture responseFuture) { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- AckResult ackResult = new AckResult(); ++- if (ResponseCode.SUCCESS == response.getCode()) { ++- ackResult.setStatus(AckStatus.OK); ++- } else { ++- ackResult.setStatus(AckStatus.NO_EXIST); ++- } ++- ackCallback.onSuccess(ackResult); ++- } catch (Exception e) { ++- ackCallback.onException(e); ++- } +++ public void operationSucceed(RemotingCommand response) { +++ AckResult ackResult = new AckResult(); +++ if (ResponseCode.SUCCESS == response.getCode()) { +++ ackResult.setStatus(AckStatus.OK); ++ } else { ++- if (!responseFuture.isSendRequestOK()) { ++- ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, ++- responseFuture.getCause())); ++- } else { ++- ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeOut + ". Request: " + request, responseFuture.getCause())); ++- } +++ ackResult.setStatus(AckStatus.NO_EXIST); ++ } +++ ackCallback.onSuccess(ackResult); +++ } ++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ ackCallback.onException(throwable); ++ } ++ }); ++ } ++@@ -999,39 +989,37 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ final AckCallback ackCallback ++ ) throws RemotingException, MQBrokerException, InterruptedException { ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader); ++- this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { +++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++- public void onComplete(ResponseFuture responseFuture) { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); ++- AckResult ackResult = new AckResult(); ++- if (ResponseCode.SUCCESS == response.getCode()) { ++- ackResult.setStatus(AckStatus.OK); ++- ackResult.setPopTime(responseHeader.getPopTime()); ++- ackResult.setExtraInfo(ExtraInfoUtil ++- .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), ++- responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR ++- + requestHeader.getOffset()); ++- } else { ++- ackResult.setStatus(AckStatus.NO_EXIST); ++- } ++- ackCallback.onSuccess(ackResult); ++- } catch (Exception e) { ++- ackCallback.onException(e); ++- } ++- } else { ++- if (!responseFuture.isSendRequestOK()) { ++- ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, ++- responseFuture.getCause())); +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ try { +++ ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); +++ AckResult ackResult = new AckResult(); +++ if (ResponseCode.SUCCESS == response.getCode()) { +++ ackResult.setStatus(AckStatus.OK); +++ ackResult.setPopTime(responseHeader.getPopTime()); +++ ackResult.setExtraInfo(ExtraInfoUtil +++ .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), +++ responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR +++ + requestHeader.getOffset()); ++ } else { ++- ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); +++ ackResult.setStatus(AckStatus.NO_EXIST); ++ } +++ ackCallback.onSuccess(ackResult); +++ } catch (Exception e) { +++ ackCallback.onException(e); ++ } ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ ackCallback.onException(throwable); +++ } ++ }); ++ } ++ ++@@ -1044,26 +1032,23 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); ++- assert pullResult != null; ++- pullCallback.onSuccess(pullResult); ++- } catch (Exception e) { ++- pullCallback.onException(e); ++- } ++- } else { ++- if (!responseFuture.isSendRequestOK()) { ++- pullCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- pullCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, ++- responseFuture.getCause())); ++- } else { ++- pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); ++- } +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ try { +++ PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); +++ pullCallback.onSuccess(pullResult); +++ } catch (Exception e) { +++ pullCallback.onException(e); ++ } ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ pullCallback.onException(throwable); +++ } ++ }); ++ } ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++index d7c8ef8d9..f3102e175 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++@@ -30,7 +30,6 @@ import org.apache.rocketmq.client.consumer.PullCallback; ++ import org.apache.rocketmq.client.consumer.PullResult; ++ import org.apache.rocketmq.client.consumer.PullStatus; ++ import org.apache.rocketmq.client.exception.MQBrokerException; ++-import org.apache.rocketmq.client.exception.MQClientException; ++ import org.apache.rocketmq.client.exception.OffsetNotFoundException; ++ import org.apache.rocketmq.client.impl.ClientRemotingProcessor; ++ import org.apache.rocketmq.client.impl.CommunicationMode; ++@@ -47,6 +46,7 @@ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +++import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.remoting.exception.RemotingCommandException; ++ import org.apache.rocketmq.remoting.netty.NettyClientConfig; ++@@ -106,19 +106,6 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ return false; ++ } ++ ++- protected static MQClientException processNullResponseErr(ResponseFuture responseFuture) { ++- MQClientException ex; ++- if (!responseFuture.isSendRequestOK()) { ++- ex = new MQClientException("send request failed", responseFuture.getCause()); ++- } else if (responseFuture.isTimeout()) { ++- ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", ++- responseFuture.getCause()); ++- } else { ++- ex = new MQClientException("unknown reason", responseFuture.getCause()); ++- } ++- return ex; ++- } ++- ++ public CompletableFuture sendHeartbeatOneway( ++ String brokerAddr, ++ HeartbeatData heartbeatData, ++@@ -146,24 +133,15 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ request.setLanguage(clientConfig.getLanguage()); ++ request.setBody(heartbeatData.encode()); ++ ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (ResponseCode.SUCCESS == response.getCode()) { ++- future.complete(response.getVersion()); ++- } else { ++- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); ++- } ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); ++- } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ if (ResponseCode.SUCCESS == response.getCode()) { +++ future0.complete(response.getVersion()); +++ } else { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture sendMessageAsync( ++@@ -177,24 +155,15 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2); ++ request.setBody(msg.getBody()); ++ ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- future.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); ++- } catch (Exception e) { ++- future.completeExceptionally(e); ++- } ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); ++- } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ try { +++ future0.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); +++ } catch (Exception e) { +++ future0.completeExceptionally(e); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture sendMessageAsync( ++@@ -216,17 +185,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ msgBatch.setBody(body); ++ ++ request.setBody(body); ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- future.complete(this.processSendResponse(brokerName, msgBatch, response, brokerAddr)); ++- } catch (Exception e) { ++- future.completeExceptionally(e); ++- } ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ try { +++ future0.complete(processSendResponse(brokerName, msgBatch, response, brokerAddr)); +++ } catch (Exception e) { +++ future0.completeExceptionally(e); ++ } +++ return future0; ++ }); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++@@ -240,21 +206,7 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ long timeoutMillis ++ ) { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); ++- ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- future.complete(response); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); ++- } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis); ++ } ++ ++ public CompletableFuture popMessageAsync( ++@@ -402,38 +354,31 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ QueryConsumerOffsetRequestHeader requestHeader, ++ long timeoutMillis ++ ) { ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- switch (response.getCode()) { ++- case ResponseCode.SUCCESS: { ++- try { ++- QueryConsumerOffsetResponseHeader responseHeader = ++- (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); ++- future.complete(responseHeader.getOffset()); ++- } catch (RemotingCommandException e) { ++- future.completeExceptionally(e); ++- } ++- break; ++- } ++- case ResponseCode.QUERY_NOT_FOUND: { ++- future.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); ++- break; ++- } ++- default: ++- break; +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ switch (response.getCode()) { +++ case ResponseCode.SUCCESS: { +++ try { +++ QueryConsumerOffsetResponseHeader responseHeader = +++ (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); +++ future0.complete(responseHeader.getOffset()); +++ } catch (RemotingCommandException e) { +++ future0.completeExceptionally(e); ++ } ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ break; ++ } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ case ResponseCode.QUERY_NOT_FOUND: { +++ future0.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); +++ break; +++ } +++ default: { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ break; +++ } +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture updateConsumerOffsetOneWay( ++@@ -461,9 +406,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ ++ CompletableFuture> future = new CompletableFuture<>(); ++ try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { +++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ switch (response.getCode()) { ++ case ResponseCode.SUCCESS: { ++ if (response.getBody() != null) { ++@@ -485,8 +435,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ break; ++ } ++ future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++ }); ++ } catch (Throwable t) { ++@@ -501,9 +454,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { +++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ if (ResponseCode.SUCCESS == response.getCode()) { ++ try { ++ GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); ++@@ -513,8 +471,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ } ++ } ++ future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++ }); ++ } catch (Throwable t) { ++@@ -529,9 +490,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { +++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ if (ResponseCode.SUCCESS == response.getCode()) { ++ try { ++ GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); ++@@ -541,8 +507,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ } ++ } ++ future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++ }); ++ } catch (Throwable t) { ++@@ -555,57 +524,41 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ long timeoutMillis) { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader); ++ ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- try { ++- SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); ++- future.complete(responseHeader.getOffset()); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- } ++- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ try { +++ SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); +++ future0.complete(responseHeader.getOffset()); +++ } catch (Throwable t) { +++ future0.completeExceptionally(t); ++ } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ } else { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture> lockBatchMQWithFuture(String brokerAddr, ++ LockBatchRequestBody requestBody, long timeoutMillis) { ++- CompletableFuture> future = new CompletableFuture<>(); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); ++ request.setBody(requestBody.encode()); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- try { ++- LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); ++- Set messageQueues = responseBody.getLockOKMQSet(); ++- future.complete(messageQueues); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- } ++- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture> future0 = new CompletableFuture<>(); +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ try { +++ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); +++ Set messageQueues = responseBody.getLockOKMQSet(); +++ future0.complete(messageQueues); +++ } catch (Throwable t) { +++ future0.completeExceptionally(t); ++ } ++- }); ++- } catch (Exception e) { ++- future.completeExceptionally(e); ++- } ++- return future; +++ } else { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture unlockBatchMQOneway(String brokerAddr, ++@@ -624,25 +577,21 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ ++ public CompletableFuture notification(String brokerAddr, NotificationRequestHeader requestHeader, ++ long timeoutMillis) { ++- CompletableFuture future = new CompletableFuture<>(); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFICATION, requestHeader); ++- try { ++- this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenAccept(response -> { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- try { ++- NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); ++- future.complete(responseHeader.isHasMsg()); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- } else { ++- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ try { +++ NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); +++ future0.complete(responseHeader.isHasMsg()); +++ } catch (Throwable t) { +++ future0.completeExceptionally(t); ++ } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ } else { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) { ++diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java ++index d13f2cfe4..c152d38ea 100644 ++--- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java ++@@ -212,7 +212,7 @@ public class MQClientAPIImplTest { ++ RemotingCommand request = mock.getArgument(1); ++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++ responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++@@ -386,7 +386,7 @@ public class MQClientAPIImplTest { ++ RemotingCommand request = mock.getArgument(1); ++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++ responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class)); ++@@ -472,7 +472,7 @@ public class MQClientAPIImplTest { ++ message.putUserProperty("key", "value"); ++ response.setBody(MessageDecoder.encode(message, false)); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++@@ -543,7 +543,7 @@ public class MQClientAPIImplTest { ++ message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0)); ++ response.setBody(MessageDecoder.encode(message, false)); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++@@ -585,7 +585,7 @@ public class MQClientAPIImplTest { ++ response.setOpaque(request.getOpaque()); ++ response.setCode(ResponseCode.SUCCESS); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++@@ -622,7 +622,7 @@ public class MQClientAPIImplTest { ++ responseHeader.setPopTime(System.currentTimeMillis()); ++ responseHeader.setInvisibleTime(10 * 1000L); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++index fe07090d5..3227d1e1c 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++@@ -26,7 +26,6 @@ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.acl.AccessValidator; ++-import org.apache.rocketmq.client.exception.MQClientException; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.common.thread.ThreadPoolMonitor; ++@@ -51,10 +50,12 @@ import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager; ++ import org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline; ++ import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline; ++ import org.apache.rocketmq.remoting.ChannelEventListener; +++import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.RemotingServer; ++ import org.apache.rocketmq.remoting.netty.NettyRemotingServer; ++ import org.apache.rocketmq.remoting.netty.NettyServerConfig; ++ import org.apache.rocketmq.remoting.netty.RequestTask; +++import org.apache.rocketmq.remoting.netty.ResponseFuture; ++ import org.apache.rocketmq.remoting.netty.TlsSystemConfig; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.RequestCode; ++@@ -239,12 +240,21 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu ++ long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++- this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, responseFuture -> { ++- if (responseFuture.getResponseCommand() == null) { ++- future.completeExceptionally(new MQClientException("response is null after send request to client", responseFuture.getCause())); ++- return; +++ this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ future.complete(response); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++- future.complete(responseFuture.getResponseCommand()); ++ }); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++index 3f3a4ae40..e2d05b0f5 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++@@ -24,6 +24,7 @@ import java.util.ArrayList; ++ import java.util.List; ++ import java.util.Set; ++ import java.util.UUID; +++import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ThreadLocalRandom; ++ import java.util.concurrent.atomic.AtomicReference; ++ import java.util.stream.Collectors; ++@@ -85,6 +86,7 @@ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.Mockito.doAnswer; +++import static org.mockito.Mockito.doReturn; ++ ++ @RunWith(MockitoJUnitRunner.class) ++ public class MQClientAPIExtTest { ++@@ -109,13 +111,9 @@ public class MQClientAPIExtTest { ++ ++ @Test ++ public void testSendHeartbeatAsync() throws Exception { ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ assertNotNull(mqClientAPI.sendHeartbeatAsync(BROKER_ADDR, new HeartbeatData(), TIMEOUT).get()); ++ } ++@@ -123,20 +121,16 @@ public class MQClientAPIExtTest { ++ @Test ++ public void testSendMessageAsync() throws Exception { ++ AtomicReference msgIdRef = new AtomicReference<>(); ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); ++- SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); ++- sendMessageResponseHeader.setMsgId(msgIdRef.get()); ++- sendMessageResponseHeader.setQueueId(0); ++- sendMessageResponseHeader.setQueueOffset(1L); ++- response.setCode(ResponseCode.SUCCESS); ++- response.makeCustomHeaderToNet(); ++- responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); +++ SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); +++ sendMessageResponseHeader.setMsgId(msgIdRef.get()); +++ sendMessageResponseHeader.setQueueId(0); +++ sendMessageResponseHeader.setQueueOffset(1L); +++ response.setCode(ResponseCode.SUCCESS); +++ response.makeCustomHeaderToNet(); +++ future.complete(response); +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ MessageExt messageExt = createMessage(); ++ msgIdRef.set(MessageClientIDSetter.getUniqID(messageExt)); ++@@ -150,20 +144,16 @@ public class MQClientAPIExtTest { ++ ++ @Test ++ public void testSendMessageListAsync() throws Exception { ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); ++- SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); ++- sendMessageResponseHeader.setMsgId(""); ++- sendMessageResponseHeader.setQueueId(0); ++- sendMessageResponseHeader.setQueueOffset(1L); ++- response.setCode(ResponseCode.SUCCESS); ++- response.makeCustomHeaderToNet(); ++- responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); +++ SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); +++ sendMessageResponseHeader.setMsgId(""); +++ sendMessageResponseHeader.setQueueId(0); +++ sendMessageResponseHeader.setQueueOffset(1L); +++ response.setCode(ResponseCode.SUCCESS); +++ response.makeCustomHeaderToNet(); +++ future.complete(response); +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ List messageExtList = new ArrayList<>(); ++ StringBuilder sb = new StringBuilder(); ++@@ -182,13 +172,9 @@ public class MQClientAPIExtTest { ++ ++ @Test ++ public void testSendMessageBackAsync() throws Exception { ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ RemotingCommand remotingCommand = mqClientAPI.sendMessageBackAsync(BROKER_ADDR, new ConsumerSendMsgBackRequestHeader(), TIMEOUT) ++ .get(); ++@@ -285,7 +271,7 @@ public class MQClientAPIExtTest { ++ body.setConsumerIdList(clientIds); ++ response.setBody(body.encode()); ++ responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); +++ invokeCallback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ ++@@ -302,7 +288,7 @@ public class MQClientAPIExtTest { ++ response.setCode(ResponseCode.SYSTEM_ERROR); ++ response.makeCustomHeaderToNet(); ++ responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); +++ invokeCallback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ ++@@ -322,7 +308,7 @@ public class MQClientAPIExtTest { ++ response.setCode(ResponseCode.SUCCESS); ++ response.makeCustomHeaderToNet(); ++ responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); +++ invokeCallback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ ++@@ -335,18 +321,15 @@ public class MQClientAPIExtTest { ++ @Test ++ public void testSearchOffsetAsync() throws Exception { ++ long offset = ThreadLocalRandom.current().nextLong(); ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); ++- SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); ++- responseHeader.setOffset(offset); ++- response.setCode(ResponseCode.SUCCESS); ++- response.makeCustomHeaderToNet(); ++- responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); +++ SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); +++ responseHeader.setOffset(offset); +++ response.setCode(ResponseCode.SUCCESS); +++ response.makeCustomHeaderToNet(); +++ future.complete(response); +++ +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader(); ++ requestHeader.setTopic(TOPIC); ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java ++index ce78fa923..6be491745 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java ++@@ -17,7 +17,22 @@ ++ package org.apache.rocketmq.remoting; ++ ++ import org.apache.rocketmq.remoting.netty.ResponseFuture; +++import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ ++ public interface InvokeCallback { +++ /** +++ * This method is expected to be invoked after {@link #operationSucceed(RemotingCommand)} +++ * or {@link #operationFail(Throwable)} +++ * +++ * @param responseFuture the returned object contains response or exception +++ */ ++ void operationComplete(final ResponseFuture responseFuture); +++ +++ default void operationSucceed(final RemotingCommand response) { +++ +++ } +++ +++ default void operationFail(final Throwable throwable) { +++ +++ } ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java ++index ff0b3df95..c8389eedb 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java ++@@ -20,11 +20,11 @@ import java.util.List; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutorService; ++ import org.apache.rocketmq.remoting.exception.RemotingConnectException; ++-import org.apache.rocketmq.remoting.exception.RemotingException; ++ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; ++ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +++import org.apache.rocketmq.remoting.netty.ResponseFuture; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ ++ public interface RemotingClient extends RemotingService { ++@@ -51,18 +51,21 @@ public interface RemotingClient extends RemotingService { ++ final long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++- invokeAsync(addr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { +++ invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { +++ +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ future.complete(response); ++- } else { ++- if (!responseFuture.isSendRequestOK()) { ++- future.completeExceptionally(new RemotingSendRequestException(addr, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- future.completeExceptionally(new RemotingTimeoutException(addr, timeoutMillis, responseFuture.getCause())); ++- } else { ++- future.completeExceptionally(new RemotingException(request.toString(), responseFuture.getCause())); ++- } +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++ }); ++ } catch (Throwable t) { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++index fce2de267..12e66f913 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++@@ -23,20 +23,23 @@ import io.netty.handler.ssl.SslContext; ++ import io.netty.handler.ssl.SslHandler; ++ import io.netty.util.concurrent.Future; ++ import io.opentelemetry.api.common.AttributesBuilder; ++-import java.net.SocketAddress; ++ import java.util.ArrayList; ++ import java.util.HashMap; ++ import java.util.Iterator; ++ import java.util.LinkedList; ++ import java.util.List; ++ import java.util.Map.Entry; +++import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; +++import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.RejectedExecutionException; ++ import java.util.concurrent.Semaphore; ++ import java.util.concurrent.TimeUnit; +++import java.util.concurrent.TimeoutException; +++import java.util.concurrent.atomic.AtomicReference; ++ import java.util.function.Consumer; ++ import javax.annotation.Nullable; ++ import org.apache.rocketmq.common.AbortProcessException; ++@@ -125,7 +128,7 @@ public abstract class NettyRemotingAbstract { ++ * Constructor, specifying capacity of one-way and asynchronous semaphores. ++ * ++ * @param permitsOneway Number of permits for one-way requests. ++- * @param permitsAsync Number of permits for asynchronous requests. +++ * @param permitsAsync Number of permits for asynchronous requests. ++ */ ++ public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) { ++ this.semaphoreOneway = new Semaphore(permitsOneway, true); ++@@ -367,8 +370,7 @@ public abstract class NettyRemotingAbstract { ++ responseFuture.release(); ++ } ++ } else { ++- log.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); ++- log.warn(cmd.toString()); +++ log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); ++ } ++ } ++ ++@@ -467,57 +469,68 @@ public abstract class NettyRemotingAbstract { ++ public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request, ++ final long timeoutMillis) ++ throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException { ++- //get the request id ++- final int opaque = request.getOpaque(); ++- ++ try { ++- final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, null, null); ++- this.responseTable.put(opaque, responseFuture); ++- final SocketAddress addr = channel.remoteAddress(); ++- channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { ++- if (f.isSuccess()) { ++- responseFuture.setSendRequestOK(true); ++- return; ++- } ++- ++- responseFuture.setSendRequestOK(false); ++- responseTable.remove(opaque); ++- responseFuture.setCause(f.cause()); ++- responseFuture.putResponse(null); ++- log.warn("Failed to write a request command to {}, caused by underlying I/O operation failure", addr); ++- }); +++ return invokeImpl(channel, request, timeoutMillis).thenApply(ResponseFuture::getResponseCommand) +++ .get(timeoutMillis, TimeUnit.MILLISECONDS); +++ } catch (ExecutionException e) { +++ throw new RemotingSendRequestException(channel.remoteAddress().toString(), e.getCause()); +++ } catch (TimeoutException e) { +++ throw new RemotingTimeoutException(channel.remoteAddress().toString(), timeoutMillis, e.getCause()); +++ } +++ } ++ ++- RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis); ++- if (null == responseCommand) { ++- if (responseFuture.isSendRequestOK()) { ++- throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis, ++- responseFuture.getCause()); ++- } else { ++- throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause()); ++- } +++ public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand request, +++ final long timeoutMillis) { +++ String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); +++ doBeforeRpcHooks(channelRemoteAddr, request); +++ return invoke0(channel, request, timeoutMillis).whenComplete((v, t) -> { +++ if (t == null) { +++ doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand()); ++ } ++- ++- return responseCommand; ++- } finally { ++- this.responseTable.remove(opaque); ++- } +++ }); ++ } ++ ++- public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, ++- final InvokeCallback invokeCallback) ++- throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { +++ protected CompletableFuture invoke0(final Channel channel, final RemotingCommand request, +++ final long timeoutMillis) { +++ CompletableFuture future = new CompletableFuture<>(); ++ long beginStartTime = System.currentTimeMillis(); ++ final int opaque = request.getOpaque(); ++- boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); +++ +++ boolean acquired; +++ try { +++ acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ return future; +++ } ++ if (acquired) { ++ final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); ++ long costTime = System.currentTimeMillis() - beginStartTime; ++ if (timeoutMillis < costTime) { ++ once.release(); ++- throw new RemotingTimeoutException("invokeAsyncImpl call timeout"); +++ future.completeExceptionally(new RemotingTimeoutException("invokeAsyncImpl call timeout")); +++ return future; ++ } ++ ++- final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once); +++ AtomicReference responseFutureReference = new AtomicReference<>(); +++ final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, request, timeoutMillis - costTime, +++ new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ future.complete(responseFutureReference.get()); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); +++ } +++ }, once); +++ responseFutureReference.set(responseFuture); ++ this.responseTable.put(opaque, responseFuture); ++ try { ++ channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { ++@@ -528,15 +541,17 @@ public abstract class NettyRemotingAbstract { ++ requestFail(opaque); ++ log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); ++ }); +++ return future; ++ } catch (Exception e) { ++ responseTable.remove(opaque); ++ responseFuture.release(); ++ log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); ++- throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); +++ future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e)); +++ return future; ++ } ++ } else { ++ if (timeoutMillis <= 0) { ++- throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast"); +++ future.completeExceptionally(new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast")); ++ } else { ++ String info = ++ String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", ++@@ -545,11 +560,31 @@ public abstract class NettyRemotingAbstract { ++ this.semaphoreAsync.availablePermits() ++ ); ++ log.warn(info); ++- throw new RemotingTimeoutException(info); +++ future.completeExceptionally(new RemotingTimeoutException(info)); ++ } +++ return future; ++ } ++ } ++ +++ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, +++ final InvokeCallback invokeCallback) { +++ invokeImpl(channel, request, timeoutMillis) +++ .whenComplete((v, t) -> { +++ if (t == null) { +++ invokeCallback.operationComplete(v); +++ } else { +++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, timeoutMillis, null, null); +++ responseFuture.setCause(t); +++ invokeCallback.operationComplete(responseFuture); +++ } +++ }) +++ .thenAccept(responseFuture -> invokeCallback.operationSucceed(responseFuture.getResponseCommand())) +++ .exceptionally(t -> { +++ invokeCallback.operationFail(t); +++ return null; +++ }); +++ } +++ ++ private void requestFail(final int opaque) { ++ ResponseFuture responseFuture = responseTable.remove(opaque); ++ if (responseFuture != null) { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++index 64621dd6c..d784351a5 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++@@ -527,15 +527,13 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ if (channel != null && channel.isActive()) { ++ long left = timeoutMillis; ++ try { ++- doBeforeRpcHooks(channelRemoteAddr, request); ++ long costTime = System.currentTimeMillis() - beginStartTime; ++ left -= costTime; ++ if (left <= 0) { ++ throw new RemotingTimeoutException("invokeSync call the addr[" + channelRemoteAddr + "] timeout"); ++ } ++ RemotingCommand response = this.invokeSyncImpl(channel, request, left); ++- doAfterRpcHooks(channelRemoteAddr, request, response); ++- this.updateChannelLastResponseTime(addr); +++ updateChannelLastResponseTime(addr); ++ return response; ++ } catch (RemotingSendRequestException e) { ++ LOGGER.warn("invokeSync: send request exception, so close the channel[{}]", channelRemoteAddr); ++@@ -727,18 +725,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ final Channel channel = this.getAndCreateChannel(addr); ++ String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); ++ if (channel != null && channel.isActive()) { ++- try { ++- doBeforeRpcHooks(channelRemoteAddr, request); ++- long costTime = System.currentTimeMillis() - beginStartTime; ++- if (timeoutMillis < costTime) { ++- throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); ++- } ++- this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); ++- } catch (RemotingSendRequestException e) { ++- LOGGER.warn("invokeAsync: send request exception, so close the channel[{}]", channelRemoteAddr); ++- this.closeChannel(addr, channel); ++- throw e; +++ long costTime = System.currentTimeMillis() - beginStartTime; +++ if (timeoutMillis < costTime) { +++ throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); ++ } +++ this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); ++ } else { ++ this.closeChannel(addr, channel); ++ throw new RemotingConnectException(addr); ++@@ -931,11 +922,19 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++- if (responseFuture != null && responseFuture.isSendRequestOK() && responseFuture.getResponseCommand() != null) { ++- NettyRemotingClient.this.updateChannelLastResponseTime(addr); ++- } ++ this.invokeCallback.operationComplete(responseFuture); ++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ updateChannelLastResponseTime(addr); +++ this.invokeCallback.operationSucceed(response); +++ } +++ +++ @Override +++ public void operationFail(final Throwable throwable) { +++ this.invokeCallback.operationFail(throwable); +++ } ++ } ++ ++ class NettyClientHandler extends SimpleChannelInboundHandler { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java ++index 19f705d74..0882818fe 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java ++@@ -22,6 +22,9 @@ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicBoolean; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; +++import org.apache.rocketmq.remoting.exception.RemotingException; +++import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +++import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ ++ public class ResponseFuture { ++@@ -59,6 +62,18 @@ public class ResponseFuture { ++ public void executeInvokeCallback() { ++ if (invokeCallback != null) { ++ if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) { +++ RemotingCommand response = getResponseCommand(); +++ if (response != null) { +++ invokeCallback.operationSucceed(response); +++ } else { +++ if (!isSendRequestOK()) { +++ invokeCallback.operationFail(new RemotingSendRequestException(channel.remoteAddress().toString(), getCause())); +++ } else if (isTimeout()) { +++ invokeCallback.operationFail(new RemotingTimeoutException(channel.remoteAddress().toString(), getTimeoutMillis(), getCause())); +++ } else { +++ invokeCallback.operationFail(new RemotingException(getRequestCommand().toString(), getCause())); +++ } +++ } ++ invokeCallback.operationComplete(this); ++ } ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java ++index 133e0ed31..5328e8845 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java ++@@ -160,31 +160,38 @@ public class RpcClientImpl implements RpcClient { ++ InvokeCallback callback = new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++- RemotingCommand responseCommand = responseFuture.getResponseCommand(); ++- if (responseCommand == null) { ++- processFailedResponse(addr, requestCommand, responseFuture, rpcResponsePromise); ++- return; ++- } +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ try { ++- switch (responseCommand.getCode()) { +++ switch (response.getCode()) { ++ case ResponseCode.SUCCESS: ++ case ResponseCode.PULL_NOT_FOUND: ++ case ResponseCode.PULL_RETRY_IMMEDIATELY: ++ case ResponseCode.PULL_OFFSET_MOVED: ++ PullMessageResponseHeader responseHeader = ++- (PullMessageResponseHeader) responseCommand.decodeCommandCustomHeader(PullMessageResponseHeader.class); ++- rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); +++ (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class); +++ rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody())); ++ default: ++- RpcResponse rpcResponse = new RpcResponse(new RpcException(responseCommand.getCode(), "unexpected remote response code")); +++ RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), "unexpected remote response code")); ++ rpcResponsePromise.setSuccess(rpcResponse); ++ ++ } ++ } catch (Exception e) { ++- String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand; ++- RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); +++ String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + requestCommand; +++ RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); ++ rpcResponsePromise.setSuccess(rpcResponse); ++ } ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ String errorMessage = "process failed. addr: " + addr + ". Request: " + requestCommand; +++ RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, throwable)); +++ rpcResponsePromise.setSuccess(rpcResponse); +++ } ++ }; ++ ++ this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback); ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java ++index 90072960b..d0da0eb2e 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java ++@@ -26,12 +26,12 @@ import org.apache.rocketmq.remoting.exception.RemotingConnectException; ++ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; ++ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++-import org.apache.rocketmq.remoting.netty.ResponseFuture; ++-import org.apache.rocketmq.remoting.netty.NettyServerConfig; ++ import org.apache.rocketmq.remoting.netty.NettyClientConfig; ++-import org.apache.rocketmq.remoting.netty.NettyRemotingServer; ++ import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +++import org.apache.rocketmq.remoting.netty.NettyRemotingServer; ++ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +++import org.apache.rocketmq.remoting.netty.NettyServerConfig; +++import org.apache.rocketmq.remoting.netty.ResponseFuture; ++ import org.apache.rocketmq.remoting.protocol.LanguageCode; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.junit.AfterClass; ++@@ -40,7 +40,6 @@ import org.junit.Test; ++ ++ import static org.assertj.core.api.Assertions.assertThat; ++ import static org.junit.Assert.assertNotNull; ++-import static org.junit.Assert.assertTrue; ++ ++ public class RemotingServerTest { ++ private static RemotingServer remotingServer; ++@@ -122,10 +121,19 @@ public class RemotingServerTest { ++ remotingClient.invokeAsync("localhost:" + remotingServer.localListenPort(), request, 1000 * 3, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ latch.countDown(); ++- assertTrue(responseFuture != null); ++- assertThat(responseFuture.getResponseCommand().getLanguage()).isEqualTo(LanguageCode.JAVA); ++- assertThat(responseFuture.getResponseCommand().getExtFields()).hasSize(2); +++ assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA); +++ assertThat(response.getExtFields()).hasSize(2); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ ++ } ++ }); ++ latch.await(); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java ++similarity index 57% ++rename from client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java ++rename to remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java ++index 80188832e..8ddcdf35d 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java ++@@ -15,23 +15,14 @@ ++ * limitations under the License. ++ */ ++ ++-package org.apache.rocketmq.client.impl; +++package org.apache.rocketmq.remoting.netty; ++ ++-import org.apache.rocketmq.remoting.InvokeCallback; ++-import org.apache.rocketmq.remoting.netty.ResponseFuture; ++- ++-public abstract class BaseInvokeCallback implements InvokeCallback { ++- private final MQClientAPIImpl mqClientAPI; ++- ++- public BaseInvokeCallback(MQClientAPIImpl mqClientAPI) { ++- this.mqClientAPI = mqClientAPI; ++- } +++import io.netty.channel.ChannelFuture; +++import io.netty.channel.local.LocalChannel; ++ +++public class MockChannel extends LocalChannel { ++ @Override ++- public void operationComplete(final ResponseFuture responseFuture) { ++- mqClientAPI.execRpcHooksAfterRequest(responseFuture); ++- onComplete(responseFuture); +++ public ChannelFuture writeAndFlush(Object msg) { +++ return new MockChannelPromise(MockChannel.this); ++ } ++- ++- public abstract void onComplete(final ResponseFuture responseFuture); ++ } ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java ++new file mode 100644 ++index 000000000..9c3a35487 ++--- /dev/null +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java ++@@ -0,0 +1,191 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.remoting.netty; +++ +++import io.netty.channel.Channel; +++import io.netty.channel.ChannelPromise; +++import io.netty.util.concurrent.Future; +++import io.netty.util.concurrent.GenericFutureListener; +++import java.util.concurrent.ExecutionException; +++import java.util.concurrent.TimeUnit; +++import java.util.concurrent.TimeoutException; +++import org.jetbrains.annotations.NotNull; +++ +++public class MockChannelPromise implements ChannelPromise { +++ protected Channel channel; +++ +++ public MockChannelPromise(Channel channel) { +++ this.channel = channel; +++ } +++ +++ @Override +++ public Channel channel() { +++ return channel; +++ } +++ +++ @Override +++ public ChannelPromise setSuccess(Void result) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise setSuccess() { +++ return this; +++ } +++ +++ @Override +++ public boolean trySuccess() { +++ return false; +++ } +++ +++ @Override +++ public ChannelPromise setFailure(Throwable cause) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise addListener(GenericFutureListener> listener) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise addListeners(GenericFutureListener>... listeners) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise removeListener(GenericFutureListener> listener) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise removeListeners(GenericFutureListener>... listeners) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise sync() throws InterruptedException { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise syncUninterruptibly() { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise await() throws InterruptedException { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise awaitUninterruptibly() { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise unvoid() { +++ return this; +++ } +++ +++ @Override +++ public boolean isVoid() { +++ return false; +++ } +++ +++ @Override +++ public boolean trySuccess(Void result) { +++ return false; +++ } +++ +++ @Override +++ public boolean tryFailure(Throwable cause) { +++ return false; +++ } +++ +++ @Override +++ public boolean setUncancellable() { +++ return false; +++ } +++ +++ @Override +++ public boolean isSuccess() { +++ return false; +++ } +++ +++ @Override +++ public boolean isCancellable() { +++ return false; +++ } +++ +++ @Override +++ public Throwable cause() { +++ return null; +++ } +++ +++ @Override +++ public boolean await(long timeout, TimeUnit unit) throws InterruptedException { +++ return false; +++ } +++ +++ @Override +++ public boolean await(long timeoutMillis) throws InterruptedException { +++ return false; +++ } +++ +++ @Override +++ public boolean awaitUninterruptibly(long timeout, TimeUnit unit) { +++ return false; +++ } +++ +++ @Override +++ public boolean awaitUninterruptibly(long timeoutMillis) { +++ return false; +++ } +++ +++ @Override +++ public Void getNow() { +++ return null; +++ } +++ +++ @Override +++ public boolean cancel(boolean mayInterruptIfRunning) { +++ return false; +++ } +++ +++ @Override +++ public boolean isCancelled() { +++ return false; +++ } +++ +++ @Override +++ public boolean isDone() { +++ return false; +++ } +++ +++ @Override +++ public Void get() throws InterruptedException, ExecutionException { +++ return null; +++ } +++ +++ @Override +++ public Void get(long timeout, +++ @NotNull java.util.concurrent.TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { +++ return null; +++ } +++} ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java ++index 8381c132b..dbbea86ea 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java ++@@ -39,9 +39,19 @@ public class NettyRemotingAbstractTest { ++ final Semaphore semaphore = new Semaphore(0); ++ ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { ++ @Override ++- public void operationComplete(final ResponseFuture responseFuture) { +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ assertThat(semaphore.availablePermits()).isEqualTo(0); ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ +++ } ++ }, new SemaphoreReleaseOnlyOnce(semaphore)); ++ ++ remotingAbstract.responseTable.putIfAbsent(1, responseFuture); ++@@ -75,9 +85,19 @@ public class NettyRemotingAbstractTest { ++ final Semaphore semaphore = new Semaphore(0); ++ ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { ++ @Override ++- public void operationComplete(final ResponseFuture responseFuture) { +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ assertThat(semaphore.availablePermits()).isEqualTo(0); ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ +++ } ++ }, new SemaphoreReleaseOnlyOnce(semaphore)); ++ ++ remotingAbstract.responseTable.putIfAbsent(1, responseFuture); ++@@ -98,7 +118,18 @@ public class NettyRemotingAbstractTest { ++ // mock timeout ++ ResponseFuture responseFuture = new ResponseFuture(null, dummyId, -1000, new InvokeCallback() { ++ @Override ++- public void operationComplete(final ResponseFuture responseFuture) { +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ ++ } ++ }, null); ++ remotingAbstract.responseTable.putIfAbsent(dummyId, responseFuture); ++@@ -111,7 +142,22 @@ public class NettyRemotingAbstractTest { ++ final Semaphore semaphore = new Semaphore(0); ++ RemotingCommand request = RemotingCommand.createRequestCommand(1, null); ++ ResponseFuture responseFuture = new ResponseFuture(null, 1, request, 3000, ++- responseFuture1 -> assertThat(semaphore.availablePermits()).isEqualTo(0), new SemaphoreReleaseOnlyOnce(semaphore)); +++ new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ assertThat(semaphore.availablePermits()).isEqualTo(0); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ +++ } +++ }, new SemaphoreReleaseOnlyOnce(semaphore)); ++ ++ remotingAbstract.responseTable.putIfAbsent(1, responseFuture); ++ RemotingCommand response = RemotingCommand.createResponseCommand(0, "Foo"); ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++index 8fabbb21d..e72e7bd53 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++@@ -16,10 +16,17 @@ ++ */ ++ package org.apache.rocketmq.remoting.netty; ++ +++import io.netty.channel.Channel; +++import io.netty.channel.ChannelFuture; +++import io.netty.channel.local.LocalChannel; ++ import java.util.concurrent.CompletableFuture; +++import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; +++import java.util.concurrent.Semaphore; ++ import org.apache.rocketmq.remoting.InvokeCallback; +++import org.apache.rocketmq.remoting.RPCHook; +++import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; ++ import org.apache.rocketmq.remoting.exception.RemotingConnectException; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; ++@@ -29,23 +36,33 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.junit.Test; ++ import org.junit.runner.RunWith; +++import org.mockito.Mock; ++ import org.mockito.Spy; ++ import org.mockito.junit.MockitoJUnitRunner; ++ ++ import static org.assertj.core.api.Assertions.assertThat; +++import static org.assertj.core.api.Assertions.assertThatThrownBy; ++ import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; ++ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; +++import static org.mockito.ArgumentMatchers.eq; ++ import static org.mockito.Mockito.doAnswer; +++import static org.mockito.Mockito.doReturn; +++import static org.mockito.Mockito.mock; +++import static org.mockito.Mockito.never; +++import static org.mockito.Mockito.times; +++import static org.mockito.Mockito.verify; ++ ++ @RunWith(MockitoJUnitRunner.class) ++ public class NettyRemotingClientTest { ++ @Spy ++ private NettyRemotingClient remotingClient = new NettyRemotingClient(new NettyClientConfig()); +++ @Mock +++ private RPCHook rpcHookMock; ++ ++ @Test ++- public void testSetCallbackExecutor() throws NoSuchFieldException, IllegalAccessException { +++ public void testSetCallbackExecutor() { ++ ExecutorService customized = Executors.newCachedThreadPool(); ++ remotingClient.setCallbackExecutor(customized); ++ assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized); ++@@ -61,7 +78,7 @@ public class NettyRemotingClientTest { ++ InvokeCallback callback = invocation.getArgument(3); ++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ ++@@ -78,9 +95,7 @@ public class NettyRemotingClientTest { ++ response.setCode(ResponseCode.SUCCESS); ++ doAnswer(invocation -> { ++ InvokeCallback callback = invocation.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++- responseFuture.setSendRequestOK(false); ++- callback.operationComplete(responseFuture); +++ callback.operationFail(new RemotingSendRequestException(null)); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ ++@@ -97,8 +112,7 @@ public class NettyRemotingClientTest { ++ response.setCode(ResponseCode.SUCCESS); ++ doAnswer(invocation -> { ++ InvokeCallback callback = invocation.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), -1L, null, null); ++- callback.operationComplete(responseFuture); +++ callback.operationFail(new RemotingTimeoutException("")); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ ++@@ -115,8 +129,7 @@ public class NettyRemotingClientTest { ++ response.setCode(ResponseCode.SUCCESS); ++ doAnswer(invocation -> { ++ InvokeCallback callback = invocation.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++- callback.operationComplete(responseFuture); +++ callback.operationFail(new RemotingException(null)); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ ++@@ -134,4 +147,158 @@ public class NettyRemotingClientTest { ++ assertThat(e.getMessage()).contains(addr); ++ } ++ } +++ +++ @Test +++ public void testInvoke0() throws ExecutionException, InterruptedException { +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ Channel channel = new MockChannel() { +++ @Override +++ public ChannelFuture writeAndFlush(Object msg) { +++ ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); +++ responseFuture.setResponseCommand(response); +++ responseFuture.executeInvokeCallback(); +++ return super.writeAndFlush(msg); +++ } +++ }; +++ CompletableFuture future = remotingClient.invoke0(channel, request, 1000L); +++ assertThat(future.get().getResponseCommand()).isEqualTo(response); +++ } +++ +++ @Test +++ public void testInvoke0WithException() { +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ Channel channel = new MockChannel() { +++ @Override +++ public ChannelFuture writeAndFlush(Object msg) { +++ ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); +++ responseFuture.executeInvokeCallback(); +++ return super.writeAndFlush(msg); +++ } +++ }; +++ CompletableFuture future = remotingClient.invoke0(channel, request, 1000L); +++ assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingException.class); +++ } +++ +++ @Test +++ public void testInvokeSync() throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { +++ remotingClient.registerRPCHook(rpcHookMock); +++ +++ Channel channel = new LocalChannel(); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); +++ responseFuture.setResponseCommand(response); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(responseFuture); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ RemotingCommand actual = remotingClient.invokeSyncImpl(channel, request, 1000); +++ assertThat(actual).isEqualTo(response); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); +++ } +++ +++ @Test +++ public void testInvokeAsync() { +++ remotingClient.registerRPCHook(rpcHookMock); +++ Channel channel = new LocalChannel(); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); +++ responseFuture.setResponseCommand(response); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(responseFuture); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ +++ InvokeCallback callback = mock(InvokeCallback.class); +++ remotingClient.invokeAsyncImpl(channel, request, 1000, callback); +++ verify(callback, times(1)).operationSucceed(eq(response)); +++ verify(callback, times(1)).operationComplete(eq(responseFuture)); +++ verify(callback, never()).operationFail(any()); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); +++ } +++ +++ @Test +++ public void testInvokeAsyncFail() { +++ remotingClient.registerRPCHook(rpcHookMock); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ +++ Channel channel = new LocalChannel(); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.completeExceptionally(new RemotingException(null)); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ +++ InvokeCallback callback = mock(InvokeCallback.class); +++ remotingClient.invokeAsyncImpl(channel, request, 1000, callback); +++ verify(callback, never()).operationSucceed(any()); +++ verify(callback, times(1)).operationComplete(any()); +++ verify(callback, times(1)).operationFail(any()); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); +++ } +++ +++ @Test +++ public void testInvokeImpl() throws ExecutionException, InterruptedException { +++ remotingClient.registerRPCHook(rpcHookMock); +++ Channel channel = new LocalChannel(); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); +++ responseFuture.setResponseCommand(response); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(responseFuture); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ +++ CompletableFuture future0 = remotingClient.invokeImpl(channel, request, 1000); +++ assertThat(future0.get()).isEqualTo(responseFuture); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); +++ } +++ +++ @Test +++ public void testInvokeImplFail() { +++ remotingClient.registerRPCHook(rpcHookMock); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ +++ Channel channel = new LocalChannel(); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.completeExceptionally(new RemotingException(null)); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ +++ assertThatThrownBy(() -> remotingClient.invokeImpl(channel, request, 1000).get()).getCause().isInstanceOf(RemotingException.class); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From b9ffe0f9576f68b8a37cf3e2f68051658ae5a9a2 Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Sun, 8 Oct 2023 16:33:44 +0800 ++Subject: [PATCH 2/7] [ISSUE #7296] Add ChannelEventListener for ++ MQClientAPIImpl (#7324) ++ ++* Add ChannelEventListener for MQClientAPIImpl ++ ++* add heartbeat when channel connect ++ ++* remove log ++ ++* Add enableHeartbeatChannelEventListener for ClientConfig ++--- ++ .../apache/rocketmq/client/ClientConfig.java | 55 ++++++++++++++----- ++ .../rocketmq/client/impl/MQClientAPIImpl.java | 9 ++- ++ .../client/impl/factory/MQClientInstance.java | 35 +++++++++++- ++ .../remoting/netty/NettyRemotingClient.java | 2 + ++ 4 files changed, 85 insertions(+), 16 deletions(-) ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++index bb0fe3522..f9843cc02 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++@@ -94,6 +94,8 @@ public class ClientConfig { ++ private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false")); ++ private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false")); ++ +++ private boolean enableHeartbeatChannelEventListener = true; +++ ++ public String buildMQClientId() { ++ StringBuilder sb = new StringBuilder(); ++ sb.append(this.getClientIP()); ++@@ -201,6 +203,7 @@ public class ClientConfig { ++ this.useHeartbeatV2 = cc.useHeartbeatV2; ++ this.startDetectorEnable = cc.startDetectorEnable; ++ this.sendLatencyEnable = cc.sendLatencyEnable; +++ this.enableHeartbeatChannelEventListener = cc.enableHeartbeatChannelEventListener; ++ this.detectInterval = cc.detectInterval; ++ this.detectTimeout = cc.detectTimeout; ++ } ++@@ -228,6 +231,7 @@ public class ClientConfig { ++ cc.enableStreamRequestType = enableStreamRequestType; ++ cc.useHeartbeatV2 = useHeartbeatV2; ++ cc.startDetectorEnable = startDetectorEnable; +++ cc.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; ++ cc.sendLatencyEnable = sendLatencyEnable; ++ cc.detectInterval = detectInterval; ++ cc.detectTimeout = detectTimeout; ++@@ -418,6 +422,14 @@ public class ClientConfig { ++ this.startDetectorEnable = startDetectorEnable; ++ } ++ +++ public boolean isEnableHeartbeatChannelEventListener() { +++ return enableHeartbeatChannelEventListener; +++ } +++ +++ public void setEnableHeartbeatChannelEventListener(boolean enableHeartbeatChannelEventListener) { +++ this.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; +++ } +++ ++ public int getDetectTimeout() { ++ return this.detectTimeout; ++ } ++@@ -444,19 +456,34 @@ public class ClientConfig { ++ ++ @Override ++ public String toString() { ++- return "ClientConfig [namesrvAddr=" + namesrvAddr ++- + ", clientIP=" + clientIP + ", instanceName=" + instanceName ++- + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads ++- + ", pollNameServerInterval=" + pollNameServerInterval ++- + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval ++- + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval ++- + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException ++- + ", unitMode=" + unitMode + ", unitName=" + unitName ++- + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS ++- + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name() ++- + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout ++- + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody ++- + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable ++- + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]"; +++ return "ClientConfig{" + +++ "namesrvAddr='" + namesrvAddr + '\'' + +++ ", clientIP='" + clientIP + '\'' + +++ ", instanceName='" + instanceName + '\'' + +++ ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + +++ ", namespace='" + namespace + '\'' + +++ ", namespaceInitialized=" + namespaceInitialized + +++ ", accessChannel=" + accessChannel + +++ ", pollNameServerInterval=" + pollNameServerInterval + +++ ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + +++ ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval + +++ ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + +++ ", unitMode=" + unitMode + +++ ", unitName='" + unitName + '\'' + +++ ", decodeReadBody=" + decodeReadBody + +++ ", decodeDecompressBody=" + decodeDecompressBody + +++ ", vipChannelEnabled=" + vipChannelEnabled + +++ ", useHeartbeatV2=" + useHeartbeatV2 + +++ ", useTLS=" + useTLS + +++ ", socksProxyConfig='" + socksProxyConfig + '\'' + +++ ", mqClientApiTimeout=" + mqClientApiTimeout + +++ ", detectTimeout=" + detectTimeout + +++ ", detectInterval=" + detectInterval + +++ ", language=" + language + +++ ", enableStreamRequestType=" + enableStreamRequestType + +++ ", sendLatencyEnable=" + sendLatencyEnable + +++ ", startDetectorEnable=" + startDetectorEnable + +++ ", enableHeartbeatChannelEventListener=" + enableHeartbeatChannelEventListener + +++ '}'; ++ } ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 2407e5737..e152be811 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -79,6 +79,7 @@ import org.apache.rocketmq.common.sysflag.PullSysFlag; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +++import org.apache.rocketmq.remoting.ChannelEventListener; ++ import org.apache.rocketmq.remoting.CommandCustomHeader; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.RPCHook; ++@@ -246,10 +247,16 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, ++ final ClientRemotingProcessor clientRemotingProcessor, ++ RPCHook rpcHook, final ClientConfig clientConfig) { +++ this(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null); +++ } +++ +++ public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, +++ final ClientRemotingProcessor clientRemotingProcessor, +++ RPCHook rpcHook, final ClientConfig clientConfig, final ChannelEventListener channelEventListener) { ++ this.clientConfig = clientConfig; ++ topAddressing = new DefaultTopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName()); ++ topAddressing.registerChangeCallBack(this); ++- this.remotingClient = new NettyRemotingClient(nettyClientConfig, null); +++ this.remotingClient = new NettyRemotingClient(nettyClientConfig, channelEventListener); ++ this.clientRemotingProcessor = clientRemotingProcessor; ++ ++ // Inject stream rpc hook first to make reserve field signature ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++index 9484b26f8..09534a176 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.client.impl.factory; ++ +++import io.netty.channel.Channel; ++ import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.HashSet; ++@@ -65,6 +66,7 @@ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.common.message.MessageQueueAssignment; ++ import org.apache.rocketmq.common.topic.TopicValidator; +++import org.apache.rocketmq.remoting.ChannelEventListener; ++ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.remoting.common.HeartbeatV2Result; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++@@ -151,7 +153,38 @@ public class MQClientInstance { ++ this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS()); ++ this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig()); ++ ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this); ++- this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig); +++ ChannelEventListener channelEventListener; +++ if (clientConfig.isEnableHeartbeatChannelEventListener()) { +++ channelEventListener = new ChannelEventListener() { +++ private final ConcurrentMap> brokerAddrTable = MQClientInstance.this.brokerAddrTable; +++ @Override +++ public void onChannelConnect(String remoteAddr, Channel channel) { +++ for (Map.Entry> addressEntry : brokerAddrTable.entrySet()) { +++ for (String address : addressEntry.getValue().values()) { +++ if (address.equals(remoteAddr)) { +++ sendHeartbeatToAllBrokerWithLockV2(false); +++ break; +++ } +++ } +++ } +++ } +++ +++ @Override +++ public void onChannelClose(String remoteAddr, Channel channel) { +++ } +++ +++ @Override +++ public void onChannelException(String remoteAddr, Channel channel) { +++ } +++ +++ @Override +++ public void onChannelIdle(String remoteAddr, Channel channel) { +++ } +++ }; +++ } else { +++ channelEventListener = null; +++ } +++ this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, channelEventListener); ++ ++ if (this.clientConfig.getNamesrvAddr() != null) { ++ this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr()); ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++index d784351a5..8631d0447 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++@@ -229,6 +229,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ handler.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); ++ } ++ +++ nettyEventExecutor.start(); +++ ++ TimerTask timerTaskScanResponseTable = new TimerTask() { ++ @Override ++ public void run(Timeout timeout) { ++-- ++2.32.0.windows.2 ++ ++ ++From 3808387e1389278edbe4ef023d200ecb3015622b Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Mon, 9 Oct 2023 16:07:56 +0800 ++Subject: [PATCH 3/7] [ISSUE #7429] clean channel map when CLIENT_UNREGISTER in ++ proxy ++ ++--- ++ .../service/sysmessage/HeartbeatSyncer.java | 31 ++++++--- ++ .../sysmessage/HeartbeatSyncerTest.java | 68 +++++++++++++++++++ ++ 2 files changed, 88 insertions(+), 11 deletions(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java ++index f70c06b8f..fee3ea87d 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java ++@@ -18,6 +18,7 @@ ++ package org.apache.rocketmq.proxy.service.sysmessage; ++ ++ import com.alibaba.fastjson.JSON; +++import io.netty.channel.Channel; ++ import java.nio.charset.StandardCharsets; ++ import java.util.List; ++ import java.util.Map; ++@@ -73,16 +74,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { ++ ); ++ this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() { ++ @Override ++- public void handle(ConsumerGroupEvent event, String s, Object... args) { ++- if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { ++- if (args == null || args.length < 1) { ++- return; ++- } ++- if (args[0] instanceof ClientChannelInfo) { ++- ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; ++- remoteChannelMap.remove(clientChannelInfo.getChannel().id().asLongText()); ++- } ++- } +++ public void handle(ConsumerGroupEvent event, String group, Object... args) { +++ processConsumerGroupEvent(event, group, args); ++ } ++ ++ @Override ++@@ -98,6 +91,18 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { ++ super.shutdown(); ++ } ++ +++ protected void processConsumerGroupEvent(ConsumerGroupEvent event, String group, Object... args) { +++ if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { +++ if (args == null || args.length < 1) { +++ return; +++ } +++ if (args[0] instanceof ClientChannelInfo) { +++ ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; +++ remoteChannelMap.remove(buildKey(group, clientChannelInfo.getChannel())); +++ } +++ } +++ } +++ ++ public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo, ++ ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, ++ Set subList) { ++@@ -189,7 +194,7 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { ++ } ++ ++ RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData()); ++- RemoteChannel channel = remoteChannelMap.computeIfAbsent(data.getGroup() + "@" + decodedChannel.id().asLongText(), key -> decodedChannel); +++ RemoteChannel channel = remoteChannelMap.computeIfAbsent(buildKey(data.getGroup(), decodedChannel), key -> decodedChannel); ++ channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute()); ++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo( ++ channel, ++@@ -228,4 +233,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { ++ // use local address, remoting port and grpc port to build unique local proxy Id ++ return proxyConfig.getLocalServeAddr() + "%" + proxyConfig.getRemotingListenPort() + "%" + proxyConfig.getGrpcServerPort(); ++ } +++ +++ private static String buildKey(String group, Channel channel) { +++ return group + "@" + channel.id().asLongText(); +++ } ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++index 43fba3d03..9a2c5e343 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++@@ -27,6 +27,7 @@ import com.google.common.collect.Sets; ++ import io.netty.channel.Channel; ++ import io.netty.channel.ChannelId; ++ import java.time.Duration; +++import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.HashSet; ++ import java.util.List; ++@@ -35,6 +36,7 @@ import java.util.concurrent.CompletableFuture; ++ import java.util.stream.Collectors; ++ import org.apache.commons.lang3.RandomStringUtils; ++ import org.apache.rocketmq.broker.client.ClientChannelInfo; +++import org.apache.rocketmq.broker.client.ConsumerGroupEvent; ++ import org.apache.rocketmq.broker.client.ConsumerManager; ++ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt; ++ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++@@ -320,6 +322,72 @@ public class HeartbeatSyncerTest extends InitConfigTest { ++ } ++ } ++ +++ @Test +++ public void testProcessConsumerGroupEventForRemoting() { +++ String consumerGroup = "consumerGroup"; +++ Channel channel = createMockChannel(); +++ RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class); +++ RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, Collections.emptySet()); +++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo( +++ remotingChannel, +++ clientId, +++ LanguageCode.JAVA, +++ 4 +++ ); +++ +++ testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); +++ } +++ +++ @Test +++ public void testProcessConsumerGroupEventForGrpcV2() { +++ String consumerGroup = "consumerGroup"; +++ GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class); +++ GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class); +++ GrpcClientChannel grpcClientChannel = new GrpcClientChannel( +++ proxyRelayService, grpcClientSettingsManager, grpcChannelManager, +++ ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress), +++ clientId); +++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo( +++ grpcClientChannel, +++ clientId, +++ LanguageCode.JAVA, +++ 5 +++ ); +++ +++ testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); +++ } +++ +++ private void testProcessConsumerGroupEvent(String consumerGroup, ClientChannelInfo clientChannelInfo) { +++ HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null); +++ SendResult okSendResult = new SendResult(); +++ okSendResult.setSendStatus(SendStatus.SEND_OK); +++ +++ ArgumentCaptor messageArgumentCaptor = ArgumentCaptor.forClass(Message.class); +++ doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt) +++ .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong()); +++ +++ heartbeatSyncer.onConsumerRegister( +++ consumerGroup, +++ clientChannelInfo, +++ ConsumeType.CONSUME_PASSIVELY, +++ MessageModel.CLUSTERING, +++ ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, +++ Collections.emptySet() +++ ); +++ await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 1); +++ +++ // change local serve addr, to simulate other proxy receive messages +++ heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10); +++ ArgumentCaptor channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class); +++ doReturn(true).when(consumerManager).registerConsumer(anyString(), channelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean()); +++ +++ heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null); +++ assertEquals(1, heartbeatSyncer.remoteChannelMap.size()); +++ +++ heartbeatSyncer.processConsumerGroupEvent(ConsumerGroupEvent.CLIENT_UNREGISTER, consumerGroup, channelInfoArgumentCaptor.getValue()); +++ assertTrue(heartbeatSyncer.remoteChannelMap.isEmpty()); +++ } +++ ++ private MessageExt convertFromMessage(Message message) { ++ MessageExt messageExt = new MessageExt(); ++ messageExt.setTopic(message.getTopic()); ++-- ++2.32.0.windows.2 ++ ++ ++From 0027a1486d4f2d6f7dce3010751167e883783945 Mon Sep 17 00:00:00 2001 ++From: redlsz ++Date: Mon, 9 Oct 2023 16:52:10 +0800 ++Subject: [PATCH 4/7] [ISSUE #7412] Fix pop revive message error when reput ++ checkpoint ++ ++--- ++ .../org/apache/rocketmq/broker/processor/PopReviveService.java | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java ++index 93167db37..d5174d3d1 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java ++@@ -595,6 +595,7 @@ public class PopReviveService extends ServiceThread { ++ newCk.setCId(oldCK.getCId()); ++ newCk.setTopic(oldCK.getTopic()); ++ newCk.setQueueId(oldCK.getQueueId()); +++ newCk.setBrokerName(oldCK.getBrokerName()); ++ newCk.addDiff(0); ++ MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId); ++ brokerController.getMessageStore().putMessage(ckMsg); ++-- ++2.32.0.windows.2 ++ ++ ++From b18e564addbcff50165a5e1d9d4ab7db789d901b Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Mon, 9 Oct 2023 21:43:01 +0800 ++Subject: [PATCH 5/7] [ISSUE #7431] Fix flaky test of ++ DLedgerControllerTest#testBrokerLifecycleListener (#7432) ++ ++* Fix flaky test of DLedgerControllerTest#testBrokerLifecycleListener ++--- ++ .../impl/DLedgerControllerTest.java | 26 ++++++++++++------- ++ 1 file changed, 17 insertions(+), 9 deletions(-) ++ ++diff --git a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java ++index 595a5cb65..d6e5449c5 100644 ++--- a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java +++++ b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java ++@@ -63,7 +63,8 @@ public class DLedgerControllerTest { ++ private List baseDirs; ++ private List controllers; ++ ++- public DLedgerController launchController(final String group, final String peers, final String selfId, final boolean isEnableElectUncleanMaster) { +++ public DLedgerController launchController(final String group, final String peers, final String selfId, +++ final boolean isEnableElectUncleanMaster) { ++ String tmpdir = System.getProperty("java.io.tmpdir"); ++ final String path = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + group + File.separator + selfId; ++ baseDirs.add(path); ++@@ -121,11 +122,11 @@ public class DLedgerControllerTest { ++ final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, nextBrokerId, brokerAddress); ++ RemotingCommand remotingCommand2 = leader.registerBroker(registerBrokerToControllerRequestHeader).get(2, TimeUnit.SECONDS); ++ ++- ++ assertEquals(ResponseCode.SUCCESS, remotingCommand2.getCode()); ++ } ++ ++- public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, Long brokerId, +++ public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, +++ Long brokerId, ++ boolean exceptSuccess) throws Exception { ++ final ElectMasterRequestHeader electMasterRequestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); ++ RemotingCommand command = leader.electMaster(electMasterRequestHeader).get(2, TimeUnit.SECONDS); ++@@ -186,9 +187,9 @@ public class DLedgerControllerTest { ++ registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L); ++ registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L); ++ // try elect ++- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L,true); ++- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); ++- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L,false); +++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, true); +++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); +++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, false); ++ final RemotingCommand getInfoResponse = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS); ++ final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) getInfoResponse.readCustomHeader(); ++ assertEquals(1, replicaInfo.getMasterEpoch().intValue()); ++@@ -239,6 +240,8 @@ public class DLedgerControllerTest { ++ @Test ++ public void testBrokerLifecycleListener() throws Exception { ++ final DLedgerController leader = mockMetaData(false); +++ +++ assertTrue(leader.isLeaderState()); ++ // Mock that master broker has been inactive, and try to elect a new master from sync-state-set ++ // But we shut down two controller, so the ElectMasterEvent will be appended to DLedger failed. ++ // So the statemachine still keep the stale master's information ++@@ -247,15 +250,20 @@ public class DLedgerControllerTest { ++ dLedgerController.shutdown(); ++ controllers.remove(dLedgerController); ++ } +++ ++ final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME); ++ setBrokerElectPolicy(leader, 1L); ++ Exception exception = null; +++ RemotingCommand remotingCommand = null; ++ try { ++- leader.electMaster(request).get(5, TimeUnit.SECONDS); +++ remotingCommand = leader.electMaster(request).get(5, TimeUnit.SECONDS); ++ } catch (Exception e) { ++ exception = e; ++ } ++- assertNotNull(exception); +++ +++ assertTrue(exception != null || +++ remotingCommand != null && remotingCommand.getCode() == ResponseCode.CONTROLLER_NOT_LEADER); +++ ++ // Shut down leader controller ++ leader.shutdown(); ++ controllers.remove(leader); ++@@ -272,7 +280,7 @@ public class DLedgerControllerTest { ++ setBrokerAlivePredicate(newLeader, 1L); ++ // Check if the statemachine is stale ++ final RemotingCommand resp = newLeader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)). ++- get(10, TimeUnit.SECONDS); +++ get(10, TimeUnit.SECONDS); ++ final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader(); ++ assertEquals(1, replicaInfo.getMasterBrokerId().longValue()); ++ assertEquals(1, replicaInfo.getMasterEpoch().intValue()); ++-- ++2.32.0.windows.2 ++ ++ ++From 38d3d5d95d371ac89f7d491a4c8719b4a22c60e1 Mon Sep 17 00:00:00 2001 ++From: mxsm ++Date: Tue, 10 Oct 2023 09:37:04 +0800 ++Subject: [PATCH 6/7] [ISSUE #7433]Update the version in the README.md document ++ to 5.1.4 (#7434) ++ ++--- ++ README.md | 8 ++++---- ++ 1 file changed, 4 insertions(+), 4 deletions(-) ++ ++diff --git a/README.md b/README.md ++index 56d253ce1..5aaa2ba73 100644 ++--- a/README.md +++++ b/README.md ++@@ -49,21 +49,21 @@ $ java -version ++ java version "1.8.0_121" ++ ``` ++ ++-For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip) to download the 5.1.3 RocketMQ binary release, +++For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip) to download the 5.1.4 RocketMQ binary release, ++ unpack it to your local disk, such as `D:\rocketmq`. ++ For macOS and Linux users, execute following commands: ++ ++ ```shell ++ # Download release from the Apache mirror ++-$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip +++$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip ++ ++ # Unpack the release ++-$ unzip rocketmq-all-5.1.3-bin-release.zip +++$ unzip rocketmq-all-5.1.4-bin-release.zip ++ ``` ++ ++ Prepare a terminal and change to the extracted `bin` directory: ++ ```shell ++-$ cd rocketmq-all-5.1.3-bin-release/bin +++$ cd rocketmq-all-5.1.4-bin-release/bin ++ ``` ++ ++ **1) Start NameServer** ++-- ++2.32.0.windows.2 ++ ++ ++From 4acb43ecee03e429d036e3ff4c28bd402d1b30c7 Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Tue, 10 Oct 2023 13:54:01 +0800 ++Subject: [PATCH 7/7] [ISSUE #7330] Add goaway and reconnection mechanism ++ (#7331) ++ ++* Add shutdown wait for NettyRemotingServer ++ ++* Add goaway and reconnection mechanism ++ ++* Add client version check ++ ++* Add enableTransparentRetry for NettyClientConfig ++ ++* Add enableReconnectForGoAway for NettyClientConfig ++ ++* fix unit test ++ ++* fix client version check ++--- ++ .../remoting/netty/NettyClientConfig.java | 30 ++++ ++ .../remoting/netty/NettyRemotingAbstract.java | 15 ++ ++ .../remoting/netty/NettyRemotingClient.java | 153 ++++++++++++++++-- ++ .../remoting/netty/NettyRemotingServer.java | 31 ++-- ++ .../remoting/netty/NettyServerConfig.java | 19 +++ ++ .../remoting/protocol/ResponseCode.java | 2 + ++ .../netty/NettyRemotingClientTest.java | 39 ++--- ++ 7 files changed, 239 insertions(+), 50 deletions(-) ++ ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java ++index b2e7df754..c28288786 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java ++@@ -53,6 +53,12 @@ public class NettyClientConfig { ++ private boolean disableCallbackExecutor = false; ++ private boolean disableNettyWorkerGroup = false; ++ +++ private long maxReconnectIntervalTimeSeconds = 60; +++ +++ private boolean enableReconnectForGoAway = true; +++ +++ private boolean enableTransparentRetry = true; +++ ++ public boolean isClientCloseSocketIfTimeout() { ++ return clientCloseSocketIfTimeout; ++ } ++@@ -181,6 +187,30 @@ public class NettyClientConfig { ++ this.disableNettyWorkerGroup = disableNettyWorkerGroup; ++ } ++ +++ public long getMaxReconnectIntervalTimeSeconds() { +++ return maxReconnectIntervalTimeSeconds; +++ } +++ +++ public void setMaxReconnectIntervalTimeSeconds(long maxReconnectIntervalTimeSeconds) { +++ this.maxReconnectIntervalTimeSeconds = maxReconnectIntervalTimeSeconds; +++ } +++ +++ public boolean isEnableReconnectForGoAway() { +++ return enableReconnectForGoAway; +++ } +++ +++ public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) { +++ this.enableReconnectForGoAway = enableReconnectForGoAway; +++ } +++ +++ public boolean isEnableTransparentRetry() { +++ return enableTransparentRetry; +++ } +++ +++ public void setEnableTransparentRetry(boolean enableTransparentRetry) { +++ this.enableTransparentRetry = enableTransparentRetry; +++ } +++ ++ public String getSocksProxyConfig() { ++ return socksProxyConfig; ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++index 12e66f913..07ace28ea 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++@@ -40,9 +40,11 @@ import java.util.concurrent.Semaphore; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.TimeoutException; ++ import java.util.concurrent.atomic.AtomicReference; +++import java.util.concurrent.atomic.AtomicBoolean; ++ import java.util.function.Consumer; ++ import javax.annotation.Nullable; ++ import org.apache.rocketmq.common.AbortProcessException; +++import org.apache.rocketmq.common.MQVersion; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ServiceThread; ++ import org.apache.rocketmq.common.UtilAll; ++@@ -60,6 +62,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; +++import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ ++ import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_IS_LONG_POLLING; ++ import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE; ++@@ -120,6 +123,8 @@ public abstract class NettyRemotingAbstract { ++ */ ++ protected List rpcHooks = new ArrayList<>(); ++ +++ protected AtomicBoolean isShuttingDown = new AtomicBoolean(false); +++ ++ static { ++ NettyLogger.initNettyLogger(); ++ } ++@@ -264,6 +269,16 @@ public abstract class NettyRemotingAbstract { ++ ++ Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque); ++ +++ if (isShuttingDown.get()) { +++ if (cmd.getVersion() > MQVersion.Version.V5_1_4.ordinal()) { +++ final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY, +++ "please go away"); +++ response.setOpaque(opaque); +++ writeResponse(ctx.channel(), cmd, response); +++ return; +++ } +++ } +++ ++ if (pair.getObject1().rejectRequest()) { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY, ++ "[REJECTREQUEST]system busy, start flow control for a while"); ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++index 8631d0447..4bc51bd83 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++@@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty; ++ ++ import com.alibaba.fastjson.JSON; ++ import com.alibaba.fastjson.TypeReference; +++import com.google.common.base.Stopwatch; ++ import io.netty.bootstrap.Bootstrap; ++ import io.netty.buffer.PooledByteBufAllocator; ++ import io.netty.channel.Channel; ++@@ -48,6 +49,7 @@ import java.io.IOException; ++ import java.net.InetSocketAddress; ++ import java.net.SocketAddress; ++ import java.security.cert.CertificateException; +++import java.time.Duration; ++ import java.util.ArrayList; ++ import java.util.Collections; ++ import java.util.HashMap; ++@@ -57,6 +59,7 @@ import java.util.Map; ++ import java.util.Random; ++ import java.util.Set; ++ import java.util.concurrent.ArrayBlockingQueue; +++import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++@@ -66,6 +69,7 @@ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.concurrent.atomic.AtomicReference; ++ import java.util.concurrent.locks.Lock; ++ import java.util.concurrent.locks.ReentrantLock; +++import java.util.concurrent.locks.ReentrantReadWriteLock; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++@@ -82,6 +86,7 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; ++ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; +++import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.apache.rocketmq.remoting.proxy.SocksProxyConfig; ++ ++ public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { ++@@ -97,6 +102,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ private final Map proxyMap = new HashMap<>(); ++ private final ConcurrentHashMap bootstrapMap = new ConcurrentHashMap<>(); ++ private final ConcurrentMap channelTables = new ConcurrentHashMap<>(); +++ private final ConcurrentMap channelWrapperTables = new ConcurrentHashMap<>(); ++ ++ private final HashedWheelTimer timer = new HashedWheelTimer(r -> new Thread(r, "ClientHouseKeepingService")); ++ ++@@ -356,9 +362,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ this.timer.stop(); ++ ++ for (String addr : this.channelTables.keySet()) { ++- this.closeChannel(addr, this.channelTables.get(addr).getChannel()); +++ this.channelTables.get(addr).close(); ++ } ++ +++ this.channelWrapperTables.clear(); ++ this.channelTables.clear(); ++ ++ this.eventLoopGroupWorker.shutdownGracefully(); ++@@ -416,7 +423,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ } ++ ++ if (removeItemFromTable) { ++- this.channelTables.remove(addrRemote); +++ ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); +++ if (channelWrapper != null && channelWrapper.tryClose(channel)) { +++ this.channelTables.remove(addrRemote); +++ } ++ LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); ++ } ++ ++@@ -463,7 +473,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ } ++ ++ if (removeItemFromTable) { ++- this.channelTables.remove(addrRemote); +++ ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); +++ if (channelWrapper != null && channelWrapper.tryClose(channel)) { +++ this.channelTables.remove(addrRemote); +++ } ++ LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); ++ RemotingHelper.closeChannel(channel); ++ } ++@@ -511,7 +524,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ if (addr.contains(namesrvAddr)) { ++ ChannelWrapper channelWrapper = this.channelTables.get(addr); ++ if (channelWrapper != null) { ++- closeChannel(channelWrapper.getChannel()); +++ channelWrapper.close(); ++ } ++ } ++ } ++@@ -689,8 +702,9 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ ChannelFuture channelFuture = fetchBootstrap(addr) ++ .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); ++ LOGGER.info("createChannel: begin to connect remote host[{}] asynchronously", addr); ++- cw = new ChannelWrapper(channelFuture); +++ cw = new ChannelWrapper(addr, channelFuture); ++ this.channelTables.put(addr, cw); +++ this.channelWrapperTables.put(channelFuture.channel(), cw); ++ } ++ } catch (Exception e) { ++ LOGGER.error("createChannel: create channel exception", e); ++@@ -758,6 +772,64 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ } ++ } ++ +++ @Override +++ public CompletableFuture invoke(String addr, RemotingCommand request, +++ long timeoutMillis) { +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ final Channel channel = this.getAndCreateChannel(addr); +++ if (channel != null && channel.isActive()) { +++ return invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> { +++ if (t == null) { +++ updateChannelLastResponseTime(addr); +++ } +++ }).thenApply(ResponseFuture::getResponseCommand); +++ } else { +++ this.closeChannel(addr, channel); +++ future.completeExceptionally(new RemotingConnectException(addr)); +++ } +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ +++ @Override +++ public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand request, +++ final long timeoutMillis) { +++ Stopwatch stopwatch = Stopwatch.createStarted(); +++ return super.invokeImpl(channel, request, timeoutMillis).thenCompose(responseFuture -> { +++ RemotingCommand response = responseFuture.getResponseCommand(); +++ if (response.getCode() == ResponseCode.GO_AWAY) { +++ if (nettyClientConfig.isEnableReconnectForGoAway()) { +++ ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> { +++ try { +++ if (channelWrapper0.reconnect()) { +++ LOGGER.info("Receive go away from channel {}, recreate the channel", channel0); +++ channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0); +++ } +++ } catch (Throwable t) { +++ LOGGER.error("Channel {} reconnect error", channelWrapper0, t); +++ } +++ return channelWrapper0; +++ }); +++ if (channelWrapper != null) { +++ if (nettyClientConfig.isEnableTransparentRetry()) { +++ long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); +++ stopwatch.stop(); +++ RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); +++ Channel retryChannel = channelWrapper.getChannel(); +++ if (channel != retryChannel) { +++ return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration); +++ } +++ } +++ } +++ } +++ } +++ return CompletableFuture.completedFuture(responseFuture); +++ }); +++ } +++ ++ @Override ++ public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) { ++ ExecutorService executorThis = executor; ++@@ -877,30 +949,41 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ } ++ } ++ ++- static class ChannelWrapper { ++- private final ChannelFuture channelFuture; +++ class ChannelWrapper { +++ private final ReentrantReadWriteLock lock; +++ private ChannelFuture channelFuture; ++ // only affected by sync or async request, oneway is not included. +++ private ChannelFuture channelToClose; ++ private long lastResponseTime; +++ private volatile long lastReconnectTimestamp = 0L; +++ private final String channelAddress; ++ ++- public ChannelWrapper(ChannelFuture channelFuture) { +++ public ChannelWrapper(String address, ChannelFuture channelFuture) { +++ this.lock = new ReentrantReadWriteLock(); ++ this.channelFuture = channelFuture; ++ this.lastResponseTime = System.currentTimeMillis(); +++ this.channelAddress = address; ++ } ++ ++ public boolean isOK() { ++- return this.channelFuture.channel() != null && this.channelFuture.channel().isActive(); +++ return getChannel() != null && getChannel().isActive(); ++ } ++ ++ public boolean isWritable() { ++- return this.channelFuture.channel().isWritable(); +++ return getChannel().isWritable(); ++ } ++ ++ private Channel getChannel() { ++- return this.channelFuture.channel(); +++ return getChannelFuture().channel(); ++ } ++ ++ public ChannelFuture getChannelFuture() { ++- return channelFuture; +++ lock.readLock().lock(); +++ try { +++ return this.channelFuture; +++ } finally { +++ lock.readLock().unlock(); +++ } ++ } ++ ++ public long getLastResponseTime() { ++@@ -910,6 +993,52 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ public void updateLastResponseTime() { ++ this.lastResponseTime = System.currentTimeMillis(); ++ } +++ +++ public boolean reconnect() { +++ if (lock.writeLock().tryLock()) { +++ try { +++ if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) { +++ channelToClose = channelFuture; +++ String[] hostAndPort = getHostAndPort(channelAddress); +++ channelFuture = fetchBootstrap(channelAddress) +++ .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); +++ lastReconnectTimestamp = System.currentTimeMillis(); +++ return true; +++ } +++ } finally { +++ lock.writeLock().unlock(); +++ } +++ } +++ return false; +++ } +++ +++ public boolean tryClose(Channel channel) { +++ try { +++ lock.readLock().lock(); +++ if (channelFuture != null) { +++ if (channelFuture.channel().equals(channel)) { +++ return true; +++ } +++ } +++ } finally { +++ lock.readLock().unlock(); +++ } +++ return false; +++ } +++ +++ public void close() { +++ try { +++ lock.writeLock().lock(); +++ if (channelFuture != null) { +++ closeChannel(channelFuture.channel()); +++ } +++ if (channelToClose != null) { +++ closeChannel(channelToClose.channel()); +++ } +++ } finally { +++ lock.writeLock().unlock(); +++ } +++ } ++ } ++ ++ class InvokeCallbackWrapper implements InvokeCallback { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++index aa0d46542..735d36168 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++@@ -53,6 +53,19 @@ import io.netty.util.HashedWheelTimer; ++ import io.netty.util.Timeout; ++ import io.netty.util.TimerTask; ++ import io.netty.util.concurrent.DefaultEventExecutorGroup; +++import java.io.IOException; +++import java.net.InetSocketAddress; +++import java.security.cert.CertificateException; +++import java.time.Duration; +++import java.util.List; +++import java.util.NoSuchElementException; +++import java.util.concurrent.ConcurrentHashMap; +++import java.util.concurrent.ConcurrentMap; +++import java.util.concurrent.ExecutorService; +++import java.util.concurrent.Executors; +++import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ThreadPoolExecutor; +++import java.util.concurrent.TimeUnit; ++ import org.apache.commons.collections.CollectionUtils; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.Pair; ++@@ -74,19 +87,6 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ ++-import java.io.IOException; ++-import java.net.InetSocketAddress; ++-import java.security.cert.CertificateException; ++-import java.util.List; ++-import java.util.NoSuchElementException; ++-import java.util.concurrent.ConcurrentHashMap; ++-import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++-import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ThreadPoolExecutor; ++-import java.util.concurrent.TimeUnit; ++- ++ @SuppressWarnings("NullableProblems") ++ public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME); ++@@ -305,6 +305,10 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ @Override ++ public void shutdown() { ++ try { +++ if (nettyServerConfig.isEnableShutdownGracefully() && isShuttingDown.compareAndSet(false, true)) { +++ Thread.sleep(Duration.ofSeconds(nettyServerConfig.getShutdownWaitTimeSeconds()).toMillis()); +++ } +++ ++ this.timer.stop(); ++ ++ this.eventLoopGroupBoss.shutdownGracefully(); ++@@ -736,6 +740,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ ++ @Override ++ public void shutdown() { +++ isShuttingDown.set(true); ++ if (this.serverChannel != null) { ++ try { ++ this.serverChannel.close().await(5, TimeUnit.SECONDS); ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java ++index 59ef2c84f..756661f62 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java ++@@ -38,6 +38,9 @@ public class NettyServerConfig implements Cloneable { ++ private int serverSocketBacklog = NettySystemConfig.socketBacklog; ++ private boolean serverPooledByteBufAllocatorEnable = true; ++ +++ private boolean enableShutdownGracefully = false; +++ private int shutdownWaitTimeSeconds = 30; +++ ++ /** ++ * make install ++ * ++@@ -171,4 +174,20 @@ public class NettyServerConfig implements Cloneable { ++ public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) { ++ this.writeBufferHighWaterMark = writeBufferHighWaterMark; ++ } +++ +++ public boolean isEnableShutdownGracefully() { +++ return enableShutdownGracefully; +++ } +++ +++ public void setEnableShutdownGracefully(boolean enableShutdownGracefully) { +++ this.enableShutdownGracefully = enableShutdownGracefully; +++ } +++ +++ public int getShutdownWaitTimeSeconds() { +++ return shutdownWaitTimeSeconds; +++ } +++ +++ public void setShutdownWaitTimeSeconds(int shutdownWaitTimeSeconds) { +++ this.shutdownWaitTimeSeconds = shutdownWaitTimeSeconds; +++ } ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java ++index e81dadf2e..be945c48f 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java ++@@ -99,6 +99,8 @@ public class ResponseCode extends RemotingSysResponseCode { ++ public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004; ++ public static final int RPC_TIME_OUT = -1006; ++ +++ public static final int GO_AWAY = 1500; +++ ++ /** ++ * Controller response code ++ */ ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++index e72e7bd53..1cc6b4f46 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++@@ -47,7 +47,6 @@ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; ++-import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.doReturn; ++ import static org.mockito.Mockito.mock; ++ import static org.mockito.Mockito.never; ++@@ -74,13 +73,11 @@ public class NettyRemotingClientTest { ++ ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++- doAnswer(invocation -> { ++- InvokeCallback callback = invocation.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++- responseFuture.setResponseCommand(response); ++- callback.operationSucceed(responseFuture.getResponseCommand()); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); +++ responseFuture.setResponseCommand(response); +++ CompletableFuture future0 = new CompletableFuture<>(); +++ future0.complete(responseFuture.getResponseCommand()); +++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); ++ RemotingCommand actual = future.get(); ++@@ -93,11 +90,9 @@ public class NettyRemotingClientTest { ++ ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++- doAnswer(invocation -> { ++- InvokeCallback callback = invocation.getArgument(3); ++- callback.operationFail(new RemotingSendRequestException(null)); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +++ CompletableFuture future0 = new CompletableFuture<>(); +++ future0.completeExceptionally(new RemotingSendRequestException(null)); +++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); ++ Throwable thrown = catchThrowable(future::get); ++@@ -110,11 +105,9 @@ public class NettyRemotingClientTest { ++ ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++- doAnswer(invocation -> { ++- InvokeCallback callback = invocation.getArgument(3); ++- callback.operationFail(new RemotingTimeoutException("")); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +++ CompletableFuture future0 = new CompletableFuture<>(); +++ future0.completeExceptionally(new RemotingTimeoutException("")); +++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); ++ Throwable thrown = catchThrowable(future::get); ++@@ -125,13 +118,9 @@ public class NettyRemotingClientTest { ++ public void testRemotingException() throws Exception { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); ++ ++- RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- response.setCode(ResponseCode.SUCCESS); ++- doAnswer(invocation -> { ++- InvokeCallback callback = invocation.getArgument(3); ++- callback.operationFail(new RemotingException(null)); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +++ CompletableFuture future0 = new CompletableFuture<>(); +++ future0.completeExceptionally(new RemotingException("")); +++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); ++ Throwable thrown = catchThrowable(future::get); ++-- ++2.32.0.windows.2 ++ +diff --git a/patch021-backport-some-enhancements.patch b/patch021-backport-some-enhancements.patch +new file mode 100644 +index 000000000..839b65fea +--- /dev/null ++++ b/patch021-backport-some-enhancements.patch +@@ -0,0 +1,344 @@ ++From dc3f22ffe9eb83ace991b68921076093c7c0da5f Mon Sep 17 00:00:00 2001 ++From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> ++Date: Tue, 10 Oct 2023 17:39:23 +0800 ++Subject: [PATCH 1/6] add getter for class Message ,fix json serialize bug ++ (#7439) ++ ++Co-authored-by: LetLetMe ++--- ++ .../rocketmq/common/message/Message.java | 24 ++++++++++++++++++- ++ 1 file changed, 23 insertions(+), 1 deletion(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/message/Message.java b/common/src/main/java/org/apache/rocketmq/common/message/Message.java ++index e02b526a1..c7997c473 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/message/Message.java +++++ b/common/src/main/java/org/apache/rocketmq/common/message/Message.java ++@@ -218,14 +218,36 @@ public class Message implements Serializable { ++ public void setDelayTimeSec(long sec) { ++ this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC, String.valueOf(sec)); ++ } +++ +++ public long getDelayTimeSec() { +++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC); +++ if (t != null) { +++ return Long.parseLong(t); +++ } +++ return 0; +++ } +++ ++ public void setDelayTimeMs(long timeMs) { ++ this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_MS, String.valueOf(timeMs)); ++ } +++ +++ public long getDelayTimeMs() { +++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS); +++ if (t != null) { +++ return Long.parseLong(t); +++ } +++ return 0; +++ } +++ ++ public void setDeliverTimeMs(long timeMs) { ++ this.putProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(timeMs)); ++ } ++ ++ public long getDeliverTimeMs() { ++- return Long.parseLong(this.getUserProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS)); +++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS); +++ if (t != null) { +++ return Long.parseLong(t); +++ } +++ return 0; ++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 7e4879a3bc120d6289aabc8354a2811f349ac8a6 Mon Sep 17 00:00:00 2001 ++From: fujian-zfj <2573259572@qq.com> ++Date: Wed, 11 Oct 2023 14:45:07 +0800 ++Subject: [PATCH 2/6] [ISSUE #7441] Fix log "Init the confirmOffset" keep ++ printing error in controller mode (#7442) ++ ++* typo int readme[ecosystem] ++ ++* fix keep printing log problem ++--- ++ store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++index 456bf2b86..f98e9a284 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++@@ -580,7 +580,7 @@ public class CommitLog implements Swappable { ++ return this.defaultMessageStore.getMaxPhyOffset(); ++ } ++ // First time it will compute the confirmOffset. ++- if (this.confirmOffset <= 0) { +++ if (this.confirmOffset < 0) { ++ setConfirmOffset(((AutoSwitchHAService) this.defaultMessageStore.getHaService()).computeConfirmOffset()); ++ log.info("Init the confirmOffset to {}.", this.confirmOffset); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 5d492c338258d07613103e6ae16df4c6fa5b3838 Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Fri, 13 Oct 2023 11:23:30 +0800 ++Subject: [PATCH 3/6] [ISSUE #7444] Fix testCalculateFileSizeInPath test can ++ not rerun in same environment (#7445) ++ ++* Fix testCalculateFileSizeInPath test can not rerun in same environment ++ ++* Ensure that files are always deleted ++--- ++ .../apache/rocketmq/common/UtilAllTest.java | 83 +++++++++++-------- ++ 1 file changed, 48 insertions(+), 35 deletions(-) ++ ++diff --git a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java ++index f568a65f4..a0653d7fc 100644 ++--- a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java +++++ b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java ++@@ -238,41 +238,54 @@ public class UtilAllTest { ++ */ ++ String basePath = System.getProperty("java.io.tmpdir") + File.separator + "testCalculateFileSizeInPath"; ++ File baseFile = new File(basePath); ++- // test empty path ++- assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile)); ++- ++- // create baseDir ++- assertTrue(baseFile.mkdirs()); ++- ++- File file0 = new File(baseFile, "file_0"); ++- assertTrue(file0.createNewFile()); ++- writeFixedBytesToFile(file0, 1313); ++- ++- assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile)); ++- ++- // build a file tree like above ++- File dir1 = new File(baseFile, "dir_1"); ++- dir1.mkdirs(); ++- File file10 = new File(dir1, "file_1_0"); ++- File file11 = new File(dir1, "file_1_1"); ++- File dir12 = new File(dir1, "dir_1_2"); ++- dir12.mkdirs(); ++- File file120 = new File(dir12, "file_1_2_0"); ++- File dir2 = new File(baseFile, "dir_2"); ++- dir2.mkdirs(); ++- ++- // write all file with 1313 bytes data ++- assertTrue(file10.createNewFile()); ++- writeFixedBytesToFile(file10, 1313); ++- assertTrue(file11.createNewFile()); ++- writeFixedBytesToFile(file11, 1313); ++- assertTrue(file120.createNewFile()); ++- writeFixedBytesToFile(file120, 1313); ++- ++- assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile)); ++- ++- // clear all file ++- baseFile.deleteOnExit(); +++ try { +++ // test empty path +++ assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile)); +++ +++ // create baseDir +++ assertTrue(baseFile.mkdirs()); +++ +++ File file0 = new File(baseFile, "file_0"); +++ assertTrue(file0.createNewFile()); +++ writeFixedBytesToFile(file0, 1313); +++ +++ assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile)); +++ +++ // build a file tree like above +++ File dir1 = new File(baseFile, "dir_1"); +++ dir1.mkdirs(); +++ File file10 = new File(dir1, "file_1_0"); +++ File file11 = new File(dir1, "file_1_1"); +++ File dir12 = new File(dir1, "dir_1_2"); +++ dir12.mkdirs(); +++ File file120 = new File(dir12, "file_1_2_0"); +++ File dir2 = new File(baseFile, "dir_2"); +++ dir2.mkdirs(); +++ +++ // write all file with 1313 bytes data +++ assertTrue(file10.createNewFile()); +++ writeFixedBytesToFile(file10, 1313); +++ assertTrue(file11.createNewFile()); +++ writeFixedBytesToFile(file11, 1313); +++ assertTrue(file120.createNewFile()); +++ writeFixedBytesToFile(file120, 1313); +++ +++ assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile)); +++ } finally { +++ deleteFolder(baseFile); +++ } +++ } +++ +++ public static void deleteFolder(File folder) { +++ if (folder.isDirectory()) { +++ File[] files = folder.listFiles(); +++ if (files != null) { +++ for (File file : files) { +++ deleteFolder(file); +++ } +++ } +++ } +++ folder.delete(); ++ } ++ ++ private void writeFixedBytesToFile(File file, int size) throws Exception { ++-- ++2.32.0.windows.2 ++ ++ ++From 28427d40129e3aa0c6f951535617e5cac0a8211b Mon Sep 17 00:00:00 2001 ++From: Lei Sun ++Date: Fri, 13 Oct 2023 13:42:27 +0800 ++Subject: [PATCH 4/6] [ISSUE #7425] Add RoccketmqControllerConsole log to fix ++ bug (#7458) ++ ++--- ++ .../org/apache/rocketmq/common/constant/LoggerName.java | 1 + ++ .../org/apache/rocketmq/controller/ControllerStartup.java | 7 ++++--- ++ controller/src/main/resources/rmq.controller.logback.xml | 4 ++++ ++ 3 files changed, 9 insertions(+), 3 deletions(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++index cb04b00b3..61310893f 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +++++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++@@ -21,6 +21,7 @@ public class LoggerName { ++ public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; ++ public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; ++ public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; +++ public static final String CONTROLLER_CONSOLE_NAME = "RocketmqControllerConsole"; ++ public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; ++ public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; ++ public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java ++index 401720d05..9e96a704d 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java ++@@ -94,9 +94,10 @@ public class ControllerStartup { ++ } ++ ++ if (commandLine.hasOption('p')) { ++- MixAll.printObjectProperties(null, controllerConfig); ++- MixAll.printObjectProperties(null, nettyServerConfig); ++- MixAll.printObjectProperties(null, nettyClientConfig); +++ Logger console = LoggerFactory.getLogger(LoggerName.CONTROLLER_CONSOLE_NAME); +++ MixAll.printObjectProperties(console, controllerConfig); +++ MixAll.printObjectProperties(console, nettyServerConfig); +++ MixAll.printObjectProperties(console, nettyClientConfig); ++ System.exit(0); ++ } ++ ++diff --git a/controller/src/main/resources/rmq.controller.logback.xml b/controller/src/main/resources/rmq.controller.logback.xml ++index bb158213a..18083e8f9 100644 ++--- a/controller/src/main/resources/rmq.controller.logback.xml +++++ b/controller/src/main/resources/rmq.controller.logback.xml ++@@ -116,6 +116,10 @@ ++ ++ ++ +++ +++ +++ +++ ++ ++ ++ ++-- ++2.32.0.windows.2 ++ ++ ++From dc62d7f2e1ce4f99364599f8e23d65fd88eb1cd4 Mon Sep 17 00:00:00 2001 ++From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> ++Date: Fri, 13 Oct 2023 13:45:48 +0800 ++Subject: [PATCH 5/6] [ISSUE #7451] Override toString for ++ TopicConfigAndQueueMapping ++ ++--- ++ .../statictopic/TopicConfigAndQueueMapping.java | 10 ++++++++++ ++ 1 file changed, 10 insertions(+) ++ ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java ++index c937fec23..d13692735 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.remoting.protocol.statictopic; ++ +++import org.apache.commons.lang3.StringUtils; ++ import org.apache.commons.lang3.builder.EqualsBuilder; ++ import org.apache.commons.lang3.builder.HashCodeBuilder; ++ import org.apache.rocketmq.common.TopicConfig; ++@@ -60,4 +61,13 @@ public class TopicConfigAndQueueMapping extends TopicConfig { ++ .append(mappingDetail) ++ .toHashCode(); ++ } +++ +++ @Override +++ public String toString() { +++ String string = super.toString(); +++ if (StringUtils.isNotBlank(string)) { +++ string = string.substring(0, string.length() - 1) + ", mappingDetail=" + mappingDetail + "]"; +++ } +++ return string; +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 2113fa371b9c2bf7c512f8ad234e51c616f1362c Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Fri, 13 Oct 2023 13:47:09 +0800 ++Subject: [PATCH 6/6] [ISSUE #7453] Fix the problem in constructing the ++ GetMessageResult (#7456) ++ ++* Fix the problem in constructing the GetMessageResult ++ ++* Optimize the initialization size of GetMessageResult ++--- ++ .../apache/rocketmq/broker/processor/PeekMessageProcessor.java | 3 +-- ++ .../apache/rocketmq/broker/processor/PopMessageProcessor.java | 3 +-- ++ 2 files changed, 2 insertions(+), 4 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java ++index a8358c4ff..e1e0e13e5 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java ++@@ -129,8 +129,7 @@ public class PeekMessageProcessor implements NettyRequestProcessor { ++ } ++ int randomQ = random.nextInt(100); ++ int reviveQid = randomQ % this.brokerController.getBrokerConfig().getReviveQueueNum(); ++- int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg(); ++- GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg); +++ GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums()); ++ boolean needRetry = randomQ % 5 == 0; ++ long popTime = System.currentTimeMillis(); ++ long restNum = 0; ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java ++index 441f7de08..0d9bdf143 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java ++@@ -347,8 +347,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { ++ reviveQid = (int) Math.abs(ckMessageNumber.getAndIncrement() % this.brokerController.getBrokerConfig().getReviveQueueNum()); ++ } ++ ++- int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg(); ++- GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg); +++ GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums()); ++ ExpressionMessageFilter finalMessageFilter = messageFilter; ++ StringBuilder finalOrderCountInfo = orderCountInfo; ++ ++-- ++2.32.0.windows.2 ++ +diff --git a/pom.xml b/pom.xml +index 4202d4095..a3f7c2270 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -137,7 +137,7 @@ + 1.29.0-alpha + 2.0.6 + 2.20.29 +- 1.0.3 ++ 1.0.2 + 2.13.4.2 + + +@@ -713,7 +713,7 @@ + ${slf4j-api.version} + + +- io.github.aliyunmq ++ org.apache.rocketmq + rocketmq-rocksdb + ${rocksdb.version} + +diff --git a/store/pom.xml b/store/pom.xml +index e979030e8..e1e616123 100644 +--- a/store/pom.xml ++++ b/store/pom.xml +@@ -58,6 +58,10 @@ + com.google.guava + guava + ++ ++ commons-io ++ commons-io ++ + + + org.slf4j +diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +index f98e9a284..93102799b 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +@@ -60,6 +60,8 @@ import org.apache.rocketmq.store.ha.HAService; + import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; + import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.util.LibC; ++import org.rocksdb.RocksDBException; ++ + import sun.nio.ch.DirectBuffer; + + /** +@@ -299,8 +301,9 @@ public class CommitLog implements Swappable { + + /** + * When the normal exit, data recovery, all memory data have been flush ++ * @throws RocksDBException only in rocksdb mode + */ +- public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { ++ public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); + boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable(); + final List mappedFiles = this.mappedFileQueue.getMappedFiles(); +@@ -369,21 +372,22 @@ public class CommitLog implements Swappable { + this.setConfirmOffset(lastValidMsgPhyOffset); + } + +- this.mappedFileQueue.setFlushedWhere(processOffset); +- this.mappedFileQueue.setCommittedWhere(processOffset); +- this.mappedFileQueue.truncateDirtyFiles(processOffset); +- + // Clear ConsumeQueue redundant data + if (maxPhyOffsetOfConsumeQueue >= processOffset) { + log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset); + this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); + } ++ ++ this.mappedFileQueue.setFlushedWhere(processOffset); ++ this.mappedFileQueue.setCommittedWhere(processOffset); ++ this.mappedFileQueue.truncateDirtyFiles(processOffset); + } else { + // Commitlog case files are deleted + log.warn("The commitlog files are deleted, and delete the consume queue files"); + this.mappedFileQueue.setFlushedWhere(0); + this.mappedFileQueue.setCommittedWhere(0); +- this.defaultMessageStore.destroyLogics(); ++ this.defaultMessageStore.getQueueStore().destroy(); ++ this.defaultMessageStore.getQueueStore().loadAfterDestroy(); + } + } + +@@ -626,8 +630,10 @@ public class CommitLog implements Swappable { + return -1; + } + +- @Deprecated +- public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { ++ /** ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + // recover by the minimum time stamp + boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); + boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable(); +@@ -705,6 +711,9 @@ public class CommitLog implements Swappable { + } + } + ++ // only for rocksdb mode ++ this.getMessageStore().finishCommitLogDispatch(); ++ + processOffset += mappedFileOffset; + if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) { + if (this.defaultMessageStore.getConfirmOffset() < this.defaultMessageStore.getMinPhyOffset()) { +@@ -717,22 +726,24 @@ public class CommitLog implements Swappable { + } else { + this.setConfirmOffset(lastValidMsgPhyOffset); + } +- this.mappedFileQueue.setFlushedWhere(processOffset); +- this.mappedFileQueue.setCommittedWhere(processOffset); +- this.mappedFileQueue.truncateDirtyFiles(processOffset); + + // Clear ConsumeQueue redundant data + if (maxPhyOffsetOfConsumeQueue >= processOffset) { + log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset); + this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); + } ++ ++ this.mappedFileQueue.setFlushedWhere(processOffset); ++ this.mappedFileQueue.setCommittedWhere(processOffset); ++ this.mappedFileQueue.truncateDirtyFiles(processOffset); + } + // Commitlog case files are deleted + else { + log.warn("The commitlog files are deleted, and delete the consume queue files"); + this.mappedFileQueue.setFlushedWhere(0); + this.mappedFileQueue.setCommittedWhere(0); +- this.defaultMessageStore.destroyLogics(); ++ this.defaultMessageStore.getQueueStore().destroy(); ++ this.defaultMessageStore.getQueueStore().loadAfterDestroy(); + } + } + +@@ -755,7 +766,7 @@ public class CommitLog implements Swappable { + this.getMessageStore().onCommitLogAppend(msg, result, commitLogFile); + } + +- private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) { ++ private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) throws RocksDBException { + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + + int magicCode = byteBuffer.getInt(MessageDecoder.MESSAGE_MAGIC_CODE_POSITION); +@@ -763,28 +774,37 @@ public class CommitLog implements Swappable { + return false; + } + +- int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION); +- int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; +- int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornhostLength; +- long storeTimestamp = byteBuffer.getLong(msgStoreTimePos); +- if (0 == storeTimestamp) { +- return false; +- } +- +- if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() +- && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { +- if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { +- log.info("find check timestamp, {} {}", +- storeTimestamp, +- UtilAll.timeMillisToHumanString(storeTimestamp)); ++ if (this.defaultMessageStore.getMessageStoreConfig().isEnableRocksDBStore()) { ++ final long maxPhyOffsetInConsumeQueue = this.defaultMessageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue(); ++ long phyOffset = byteBuffer.getLong(MessageDecoder.MESSAGE_PHYSIC_OFFSET_POSITION); ++ if (phyOffset <= maxPhyOffsetInConsumeQueue) { ++ log.info("find check. beginPhyOffset: {}, maxPhyOffsetInConsumeQueue: {}", phyOffset, maxPhyOffsetInConsumeQueue); + return true; + } + } else { +- if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { +- log.info("find check timestamp, {} {}", +- storeTimestamp, +- UtilAll.timeMillisToHumanString(storeTimestamp)); +- return true; ++ int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION); ++ int bornHostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; ++ int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornHostLength; ++ long storeTimestamp = byteBuffer.getLong(msgStoreTimePos); ++ if (0 == storeTimestamp) { ++ return false; ++ } ++ ++ if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() ++ && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { ++ if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { ++ log.info("find check timestamp, {} {}", ++ storeTimestamp, ++ UtilAll.timeMillisToHumanString(storeTimestamp)); ++ return true; ++ } ++ } else { ++ if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { ++ log.info("find check timestamp, {} {}", ++ storeTimestamp, ++ UtilAll.timeMillisToHumanString(storeTimestamp)); ++ return true; ++ } + } + } + +@@ -958,8 +978,6 @@ public class CommitLog implements Swappable { + beginTimeInLock = 0; + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result)); + case UNKNOWN_ERROR: +- beginTimeInLock = 0; +- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); + default: + beginTimeInLock = 0; + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); +@@ -974,6 +992,8 @@ public class CommitLog implements Swappable { + if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) { + this.defaultMessageStore.increaseOffset(msg, getMessageNum(msg)); + } ++ } catch (RocksDBException e) { ++ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); + } finally { + topicQueueLock.unlock(topicQueueKey); + } +@@ -997,7 +1017,7 @@ public class CommitLog implements Swappable { + + public CompletableFuture asyncPutMessages(final MessageExtBatch messageExtBatch) { + messageExtBatch.setStoreTimestamp(System.currentTimeMillis()); +- AppendMessageResult result; ++ AppendMessageResult result = null; + + StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); + +@@ -1133,7 +1153,9 @@ public class CommitLog implements Swappable { + if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) { + this.defaultMessageStore.increaseOffset(messageExtBatch, (short) putMessageContext.getBatchSize()); + } +- } finally { ++ } catch (RocksDBException e) { ++ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); ++ } finally { + topicQueueLock.unlock(topicQueueKey); + } + +diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java +index 9d6fa6ad9..f3a7b7c5c 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java ++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java +@@ -17,6 +17,8 @@ + + package org.apache.rocketmq.store; + ++import org.rocksdb.RocksDBException; ++ + /** + * Dispatcher of commit log. + */ +@@ -25,6 +27,7 @@ public interface CommitLogDispatcher { + /** + * Dispatch messages from store to build consume queues, indexes, and filter data + * @param request dispatch message request ++ * @throws RocksDBException only in rocksdb mode + */ +- void dispatch(final DispatchRequest request); ++ void dispatch(final DispatchRequest request) throws RocksDBException; + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +index 56bee2af3..623509c8b 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +@@ -24,13 +24,12 @@ import java.util.Map; + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.MixAll; ++import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.attribute.CQType; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.message.MessageAccessor; + import org.apache.rocketmq.common.message.MessageConst; +-import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExtBrokerInner; +-import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.store.config.BrokerRole; +@@ -39,9 +38,9 @@ import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.queue.ConsumeQueueInterface; + import org.apache.rocketmq.store.queue.CqUnit; + import org.apache.rocketmq.store.queue.FileQueueLifeCycle; ++import org.apache.rocketmq.store.queue.MultiDispatch; + import org.apache.rocketmq.store.queue.QueueOffsetOperator; + import org.apache.rocketmq.store.queue.ReferredIterator; +-import org.apache.rocketmq.store.timer.TimerMessageStore; + + public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); +@@ -703,7 +702,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp()); + } + this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(request.getStoreTimestamp()); +- if (checkMultiDispatchQueue(request)) { ++ if (MultiDispatch.checkMultiDispatchQueue(this.messageStore.getMessageStoreConfig(), request)) { + multiDispatchLmqQueue(request, maxRetries); + } + return; +@@ -725,25 +724,6 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + this.messageStore.getRunningFlags().makeLogicsQueueError(); + } + +- private boolean checkMultiDispatchQueue(DispatchRequest dispatchRequest) { +- if (!this.messageStore.getMessageStoreConfig().isEnableMultiDispatch() +- || dispatchRequest.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) +- || dispatchRequest.getTopic().equals(TimerMessageStore.TIMER_TOPIC) +- || dispatchRequest.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC)) { +- return false; +- } +- Map prop = dispatchRequest.getPropertiesMap(); +- if (prop == null || prop.isEmpty()) { +- return false; +- } +- String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); +- String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); +- if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { +- return false; +- } +- return true; +- } +- + private void multiDispatchLmqQueue(DispatchRequest request, int maxRetries) { + Map prop = request.getPropertiesMap(); + String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); +@@ -765,9 +745,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + queueId = 0; + } + doDispatchLmqQueue(request, maxRetries, queueName, queueOffset, queueId); +- + } +- return; + } + + private void doDispatchLmqQueue(DispatchRequest request, int maxRetries, String queueName, long queueOffset, +@@ -802,7 +780,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), + // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. +- if (!isNeedHandleMultiDispatch(msg)) { ++ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { + return; + } + String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); +@@ -812,14 +790,14 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + Long[] queueOffsets = new Long[queues.length]; + for (int i = 0; i < queues.length; i++) { +- String key = queueKey(queues[i], msg); +- if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { ++ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { ++ String key = MultiDispatch.lmqQueueKey(queues[i]); + queueOffsets[i] = queueOffsetOperator.getLmqOffset(key); + } + } + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, + StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); +- removeWaitStorePropertyString(msg); ++ msg.removeWaitStorePropertyString(); + } + + @Override +@@ -830,7 +808,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), + // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. +- if (!isNeedHandleMultiDispatch(msg)) { ++ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { + return; + } + String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); +@@ -839,45 +817,13 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + } + String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + for (int i = 0; i < queues.length; i++) { +- String key = queueKey(queues[i], msg); +- if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { ++ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { ++ String key = MultiDispatch.lmqQueueKey(queues[i]); + queueOffsetOperator.increaseLmqOffset(key, (short) 1); + } + } + } + +- public boolean isNeedHandleMultiDispatch(MessageExtBrokerInner msg) { +- return messageStore.getMessageStoreConfig().isEnableMultiDispatch() +- && !msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) +- && !msg.getTopic().equals(TimerMessageStore.TIMER_TOPIC) +- && !msg.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); +- } +- +- public String queueKey(String queueName, MessageExtBrokerInner msgInner) { +- StringBuilder keyBuilder = new StringBuilder(); +- keyBuilder.append(queueName); +- keyBuilder.append('-'); +- int queueId = msgInner.getQueueId(); +- if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { +- queueId = 0; +- } +- keyBuilder.append(queueId); +- return keyBuilder.toString(); +- } +- +- private void removeWaitStorePropertyString(MessageExtBrokerInner msgInner) { +- if (msgInner.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) { +- // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message. +- // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it. +- String waitStoreMsgOKValue = msgInner.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); +- msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); +- // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later +- msgInner.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue); +- } else { +- msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); +- } +- } +- + private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, + final long cqOffset) { + +@@ -965,6 +911,11 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + return new ConsumeQueueIterator(sbr); + } + ++ @Override ++ public ReferredIterator iterateFrom(long startIndex, int count) { ++ return iterateFrom(startIndex); ++ } ++ + @Override + public CqUnit get(long offset) { + ReferredIterator it = iterateFrom(offset); +@@ -974,6 +925,20 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + return it.nextAndRelease(); + } + ++ @Override ++ public Pair getCqUnitAndStoreTime(long index) { ++ CqUnit cqUnit = get(index); ++ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); ++ return new Pair<>(cqUnit, messageStoreTime); ++ } ++ ++ @Override ++ public Pair getEarliestUnitAndStoreTime() { ++ CqUnit cqUnit = getEarliestUnit(); ++ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); ++ return new Pair<>(cqUnit, messageStoreTime); ++ } ++ + @Override + public CqUnit getEarliestUnit() { + /** +diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +index 02ea47f13..99a54e2d7 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +@@ -105,32 +105,35 @@ import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; + import org.apache.rocketmq.store.queue.ConsumeQueueInterface; + import org.apache.rocketmq.store.queue.ConsumeQueueStore; ++import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; + import org.apache.rocketmq.store.queue.CqUnit; + import org.apache.rocketmq.store.queue.ReferredIterator; + import org.apache.rocketmq.store.stats.BrokerStatsManager; + import org.apache.rocketmq.store.timer.TimerMessageStore; + import org.apache.rocketmq.store.util.PerfCounter; ++import org.rocksdb.RocksDBException; + + public class DefaultMessageStore implements MessageStore { +- private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ protected static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + + public final PerfCounter.Ticks perfs = new PerfCounter.Ticks(LOGGER); + + private final MessageStoreConfig messageStoreConfig; + // CommitLog +- private final CommitLog commitLog; ++ protected final CommitLog commitLog; + +- private final ConsumeQueueStore consumeQueueStore; ++ protected final ConsumeQueueStoreInterface consumeQueueStore; + + private final FlushConsumeQueueService flushConsumeQueueService; + +- private final CleanCommitLogService cleanCommitLogService; ++ protected final CleanCommitLogService cleanCommitLogService; + + private final CleanConsumeQueueService cleanConsumeQueueService; + + private final CorrectLogicOffsetService correctLogicOffsetService; + +- private final IndexService indexService; ++ protected final IndexService indexService; + + private final AllocateMappedFileService allocateMappedFileService; + +@@ -147,7 +150,7 @@ public class DefaultMessageStore implements MessageStore { + + private final TransientStorePool transientStorePool; + +- private final RunningFlags runningFlags = new RunningFlags(); ++ protected final RunningFlags runningFlags = new RunningFlags(); + private final SystemClock systemClock = new SystemClock(); + + private final ScheduledExecutorService scheduledExecutorService; +@@ -156,6 +159,7 @@ public class DefaultMessageStore implements MessageStore { + private final BrokerConfig brokerConfig; + + private volatile boolean shutdown = true; ++ protected boolean notifyMessageArriveInBatch = false; + + private StoreCheckpoint storeCheckpoint; + private TimerMessageStore timerMessageStore; +@@ -182,7 +186,7 @@ public class DefaultMessageStore implements MessageStore { + + private volatile long brokerInitMaxOffset = -1L; + +- protected List putMessageHookList = new ArrayList<>(); ++ private List putMessageHookList = new ArrayList<>(); + + private SendMessageBackHook sendMessageBackHook; + +@@ -222,12 +226,12 @@ public class DefaultMessageStore implements MessageStore { + this.commitLog = new CommitLog(this); + } + +- this.consumeQueueStore = new ConsumeQueueStore(this, this.messageStoreConfig); ++ this.consumeQueueStore = createConsumeQueueStore(); + +- this.flushConsumeQueueService = new FlushConsumeQueueService(); ++ this.flushConsumeQueueService = createFlushConsumeQueueService(); + this.cleanCommitLogService = new CleanCommitLogService(); +- this.cleanConsumeQueueService = new CleanConsumeQueueService(); +- this.correctLogicOffsetService = new CorrectLogicOffsetService(); ++ this.cleanConsumeQueueService = createCleanConsumeQueueService(); ++ this.correctLogicOffsetService = createCorrectLogicOffsetService(); + this.storeStatsService = new StoreStatsService(getBrokerIdentity()); + this.indexService = new IndexService(this); + +@@ -273,6 +277,22 @@ public class DefaultMessageStore implements MessageStore { + parseDelayLevel(); + } + ++ public ConsumeQueueStoreInterface createConsumeQueueStore() { ++ return new ConsumeQueueStore(this); ++ } ++ ++ public CleanConsumeQueueService createCleanConsumeQueueService() { ++ return new CleanConsumeQueueService(); ++ } ++ ++ public FlushConsumeQueueService createFlushConsumeQueueService() { ++ return new FlushConsumeQueueService(); ++ } ++ ++ public CorrectLogicOffsetService createCorrectLogicOffsetService() { ++ return new CorrectLogicOffsetService(); ++ } ++ + public boolean parseDelayLevel() { + HashMap timeUnitTable = new HashMap<>(); + timeUnitTable.put("s", 1000L); +@@ -305,7 +325,7 @@ public class DefaultMessageStore implements MessageStore { + } + + @Override +- public void truncateDirtyLogicFiles(long phyOffset) { ++ public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException { + this.consumeQueueStore.truncateDirty(phyOffset); + } + +@@ -393,6 +413,7 @@ public class DefaultMessageStore implements MessageStore { + + this.flushConsumeQueueService.start(); + this.commitLog.start(); ++ this.consumeQueueStore.start(); + this.storeStatsService.start(); + + if (this.haService != null) { +@@ -481,6 +502,7 @@ public class DefaultMessageStore implements MessageStore { + this.storeStatsService.shutdown(); + this.commitLog.shutdown(); + this.reputMessageService.shutdown(); ++ this.consumeQueueStore.shutdown(); + // dispatch-related services must be shut down after reputMessageService + this.indexService.shutdown(); + if (this.compactionService != null) { +@@ -515,7 +537,7 @@ public class DefaultMessageStore implements MessageStore { + + @Override + public void destroy() { +- this.destroyLogics(); ++ this.consumeQueueStore.destroy(); + this.commitLog.destroy(); + this.indexService.destroy(); + this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); +@@ -541,11 +563,6 @@ public class DefaultMessageStore implements MessageStore { + return commitLogSize + consumeQueueSize + indexFileSize; + } + +- @Override +- public void destroyLogics() { +- this.consumeQueueStore.destroy(); +- } +- + @Override + public CompletableFuture asyncPutMessage(MessageExtBrokerInner msg) { + +@@ -687,7 +704,7 @@ public class DefaultMessageStore implements MessageStore { + return commitLog; + } + +- public void truncateDirtyFiles(long offsetToTruncate) { ++ public void truncateDirtyFiles(long offsetToTruncate) throws RocksDBException { + + LOGGER.info("truncate dirty files to {}", offsetToTruncate); + +@@ -700,12 +717,12 @@ public class DefaultMessageStore implements MessageStore { + + long oldReputFromOffset = this.reputMessageService.getReputFromOffset(); + +- // truncate commitLog +- this.commitLog.truncateDirtyFiles(offsetToTruncate); +- + // truncate consume queue + this.truncateDirtyLogicFiles(offsetToTruncate); + ++ // truncate commitLog ++ this.commitLog.truncateDirtyFiles(offsetToTruncate); ++ + this.recoverTopicQueueTable(); + + if (!messageStoreConfig.isEnableBuildConsumeQueueConcurrently()) { +@@ -723,7 +740,7 @@ public class DefaultMessageStore implements MessageStore { + } + + @Override +- public boolean truncateFiles(long offsetToTruncate) { ++ public boolean truncateFiles(long offsetToTruncate) throws RocksDBException { + if (offsetToTruncate >= this.getMaxPhyOffset()) { + LOGGER.info("no need to truncate files, truncate offset is {}, max physical offset is {}", offsetToTruncate, this.getMaxPhyOffset()); + return true; +@@ -825,17 +842,19 @@ public class DefaultMessageStore implements MessageStore { + while (getResult.getBufferTotalSize() <= 0 + && nextBeginOffset < maxOffset + && cqFileNum++ < this.messageStoreConfig.getTravelCqFileNumWhenGetMessage()) { +- ReferredIterator bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset); +- +- if (bufferConsumeQueue == null) { +- status = GetMessageStatus.OFFSET_FOUND_NULL; +- nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset)); +- LOGGER.warn("consumer request topic: " + topic + "offset: " + offset + " minOffset: " + minOffset + " maxOffset: " +- + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset); +- break; +- } ++ ReferredIterator bufferConsumeQueue = null; + + try { ++ bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset, maxMsgNums); ++ ++ if (bufferConsumeQueue == null) { ++ status = GetMessageStatus.OFFSET_FOUND_NULL; ++ nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset)); ++ LOGGER.warn("consumer request topic: " + topic + ", offset: " + offset + ", minOffset: " + minOffset + ", maxOffset: " ++ + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset); ++ break; ++ } ++ + long nextPhyFileStartOffset = Long.MIN_VALUE; + while (bufferConsumeQueue.hasNext() + && nextBeginOffset < maxOffset) { +@@ -905,8 +924,13 @@ public class DefaultMessageStore implements MessageStore { + status = GetMessageStatus.FOUND; + nextPhyFileStartOffset = Long.MIN_VALUE; + } ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getMessage Failed. cid: {}, topic: {}, queueId: {}, offset: {}, minOffset: {}, maxOffset: {}, {}", ++ group, topic, queueId, offset, minOffset, maxOffset, e.getMessage()); + } finally { +- bufferConsumeQueue.release(); ++ if (bufferConsumeQueue != null) { ++ bufferConsumeQueue.release(); ++ } + } + } + +@@ -975,12 +999,12 @@ public class DefaultMessageStore implements MessageStore { + + @Override + public long getMinOffsetInQueue(String topic, int queueId) { +- ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); +- if (logic != null) { +- return logic.getMinOffsetInQueue(); ++ try { ++ return this.consumeQueueStore.getMinOffsetInQueue(topic, queueId); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getMinOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e); ++ return -1; + } +- +- return -1; + } + + @Override +@@ -997,38 +1021,27 @@ public class DefaultMessageStore implements MessageStore { + public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) { + ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { +- +- ReferredIterator bufferConsumeQueue = consumeQueue.iterateFrom(consumeQueueOffset); +- if (bufferConsumeQueue != null) { +- try { +- if (bufferConsumeQueue.hasNext()) { +- long offsetPy = bufferConsumeQueue.next().getPos(); +- return offsetPy; +- } +- } finally { +- bufferConsumeQueue.release(); +- } ++ CqUnit cqUnit = consumeQueue.get(consumeQueueOffset); ++ if (cqUnit != null) { ++ return cqUnit.getPos(); + } + } +- + return 0; + } + + @Override + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) { +- return getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER); ++ return this.getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER); + } + ++ @Override + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) { +- ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); +- if (logic != null) { +- long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType); +- // Make sure the result offset is in valid range. +- resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue()); +- resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue()); +- return resultOffset; ++ try { ++ return this.consumeQueueStore.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getOffsetInQueueByTime Failed. topic: {}, queueId: {}, timestamp: {} boundaryType: {}, {}", ++ topic, queueId, timestamp, boundaryType, e.getMessage()); + } +- + return 0; + } + +@@ -1088,6 +1101,10 @@ public class DefaultMessageStore implements MessageStore { + return StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()); + } + ++ public MessageArrivingListener getMessageArrivingListener() { ++ return messageArrivingListener; ++ } ++ + @Override + public HashMap getRuntimeInfo() { + HashMap result = this.storeStatsService.getRuntimeInfo(); +@@ -1121,7 +1138,6 @@ public class DefaultMessageStore implements MessageStore { + return this.commitLog.getMaxOffset(); + } + +- + @Override + public long getMinPhyOffset() { + return this.commitLog.getMinOffset(); +@@ -1141,7 +1157,10 @@ public class DefaultMessageStore implements MessageStore { + public long getEarliestMessageTime(String topic, int queueId) { + ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { +- return getStoreTime(logicQueue.getEarliestUnit()); ++ Pair pair = logicQueue.getEarliestUnitAndStoreTime(); ++ if (pair != null && pair.getObject2() != null) { ++ return pair.getObject2(); ++ } + } + + return -1; +@@ -1152,19 +1171,6 @@ public class DefaultMessageStore implements MessageStore { + return CompletableFuture.completedFuture(getEarliestMessageTime(topic, queueId)); + } + +- protected long getStoreTime(CqUnit result) { +- if (result != null) { +- try { +- final long phyOffset = result.getPos(); +- final int size = result.getSize(); +- long storeTime = this.getCommitLog().pickupStoreTimestamp(phyOffset, size); +- return storeTime; +- } catch (Exception e) { +- } +- } +- return -1; +- } +- + @Override + public long getEarliestMessageTime() { + long minPhyOffset = this.getMinPhyOffset(); +@@ -1179,13 +1185,16 @@ public class DefaultMessageStore implements MessageStore { + public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) { + ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { +- return getStoreTime(logicQueue.get(consumeQueueOffset)); ++ Pair pair = logicQueue.getCqUnitAndStoreTime(consumeQueueOffset); ++ if (pair != null && pair.getObject2() != null) { ++ return pair.getObject2(); ++ } + } +- + return -1; + } + +- @Override public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, ++ @Override ++ public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, + long consumeQueueOffset) { + return CompletableFuture.completedFuture(getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset)); + } +@@ -1354,6 +1363,7 @@ public class DefaultMessageStore implements MessageStore { + * If offset table is cleaned, and old messages are dispatching after the old consume queue is cleaned, + * consume queue will be created with old offset, then later message with new offset table can not be + * dispatched to consume queue. ++ * @throws RocksDBException only in rocksdb mode + */ + @Override + public int deleteTopics(final Set deleteTopics) { +@@ -1363,17 +1373,19 @@ public class DefaultMessageStore implements MessageStore { + + int deleteCount = 0; + for (String topic : deleteTopics) { +- ConcurrentMap queueTable = +- this.consumeQueueStore.getConsumeQueueTable().get(topic); ++ ConcurrentMap queueTable = this.consumeQueueStore.findConsumeQueueMap(topic); + + if (queueTable == null || queueTable.isEmpty()) { + continue; + } + + for (ConsumeQueueInterface cq : queueTable.values()) { +- this.consumeQueueStore.destroy(cq); +- LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}", +- cq.getTopic(), cq.getQueueId()); ++ try { ++ this.consumeQueueStore.destroy(cq); ++ } catch (RocksDBException e) { ++ LOGGER.error("DeleteTopic: ConsumeQueue cleans error!, topic={}, queueId={}", cq.getTopic(), cq.getQueueId(), e); ++ } ++ LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}", cq.getTopic(), cq.getQueueId()); + this.consumeQueueStore.removeTopicQueueTable(cq.getTopic(), cq.getQueueId()); + } + +@@ -1852,14 +1864,18 @@ public class DefaultMessageStore implements MessageStore { + return file.exists(); + } + +- private void recover(final boolean lastExitOK) { +- boolean recoverConcurrently = this.brokerConfig.isRecoverConcurrently(); ++ private boolean isRecoverConcurrently() { ++ return this.brokerConfig.isRecoverConcurrently() && !this.messageStoreConfig.isEnableRocksDBStore(); ++ } ++ ++ private void recover(final boolean lastExitOK) throws RocksDBException { ++ boolean recoverConcurrently = this.isRecoverConcurrently(); + LOGGER.info("message store recover mode: {}", recoverConcurrently ? "concurrent" : "normal"); + + // recover consume queue + long recoverConsumeQueueStart = System.currentTimeMillis(); + this.recoverConsumeQueue(); +- long maxPhyOffsetOfConsumeQueue = this.getMaxOffsetInConsumeQueue(); ++ long maxPhyOffsetOfConsumeQueue = this.consumeQueueStore.getMaxPhyOffsetInConsumeQueue(); + long recoverConsumeQueueEnd = System.currentTimeMillis(); + + // recover commitlog +@@ -1894,23 +1910,25 @@ public class DefaultMessageStore implements MessageStore { + return messageStoreConfig; + } + ++ @Override ++ public void finishCommitLogDispatch() { ++ // ignore ++ } ++ + @Override + public TransientStorePool getTransientStorePool() { + return transientStorePool; + } + + private void recoverConsumeQueue() { +- if (!this.brokerConfig.isRecoverConcurrently()) { ++ if (!this.isRecoverConcurrently()) { + this.consumeQueueStore.recover(); + } else { + this.consumeQueueStore.recoverConcurrently(); + } + } + +- private long getMaxOffsetInConsumeQueue() { +- return this.consumeQueueStore.getMaxOffsetInConsumeQueue(); +- } +- ++ @Override + public void recoverTopicQueueTable() { + long minPhyOffset = this.commitLog.getMinOffset(); + this.consumeQueueStore.recoverOffsetTable(minPhyOffset); +@@ -1949,13 +1967,17 @@ public class DefaultMessageStore implements MessageStore { + return runningFlags; + } + +- public void doDispatch(DispatchRequest req) { ++ public void doDispatch(DispatchRequest req) throws RocksDBException { + for (CommitLogDispatcher dispatcher : this.dispatcherList) { + dispatcher.dispatch(req); + } + } + +- public void putMessagePositionInfo(DispatchRequest dispatchRequest) { ++ /** ++ * @param dispatchRequest ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ protected void putMessagePositionInfo(DispatchRequest dispatchRequest) throws RocksDBException { + this.consumeQueueStore.putMessagePositionInfoWrapper(dispatchRequest); + } + +@@ -2054,7 +2076,7 @@ public class DefaultMessageStore implements MessageStore { + } + + @Override +- public ConsumeQueueStore getQueueStore() { ++ public ConsumeQueueStoreInterface getQueueStore() { + return consumeQueueStore; + } + +@@ -2065,7 +2087,7 @@ public class DefaultMessageStore implements MessageStore { + + @Override + public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, +- boolean isRecover, boolean isFileEnd) { ++ boolean isRecover, boolean isFileEnd) throws RocksDBException { + if (doDispatch && !isFileEnd) { + this.doDispatch(dispatchRequest); + } +@@ -2082,7 +2104,7 @@ public class DefaultMessageStore implements MessageStore { + } + + @Override +- public void assignOffset(MessageExtBrokerInner msg) { ++ public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException { + final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); + + if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { +@@ -2127,12 +2149,12 @@ public class DefaultMessageStore implements MessageStore { + class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher { + + @Override +- public void dispatch(DispatchRequest request) { ++ public void dispatch(DispatchRequest request) throws RocksDBException { + final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag()); + switch (tranType) { + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: +- DefaultMessageStore.this.putMessagePositionInfo(request); ++ putMessagePositionInfo(request); + break; + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: +@@ -2278,7 +2300,7 @@ public class DefaultMessageStore implements MessageStore { + return DefaultMessageStore.this.brokerConfig.getIdentifier() + CleanCommitLogService.class.getSimpleName(); + } + +- private boolean isTimeToDelete() { ++ protected boolean isTimeToDelete() { + String when = DefaultMessageStore.this.getMessageStoreConfig().getDeleteWhen(); + if (UtilAll.isItTimeToDo(when)) { + DefaultMessageStore.LOGGER.info("it's time to reclaim disk space, " + when); +@@ -2436,7 +2458,7 @@ public class DefaultMessageStore implements MessageStore { + } + + class CleanConsumeQueueService { +- private long lastPhysicalMinOffset = 0; ++ protected long lastPhysicalMinOffset = 0; + + public void run() { + try { +@@ -2446,7 +2468,7 @@ public class DefaultMessageStore implements MessageStore { + } + } + +- private void deleteExpiredFiles() { ++ protected void deleteExpiredFiles() { + int deleteLogicsFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval(); + + long minOffset = DefaultMessageStore.this.commitLog.getMinOffset(); +@@ -2551,7 +2573,7 @@ public class DefaultMessageStore implements MessageStore { + + if (cqUnit.getPos() >= minPhyOffset) { + +- // Normal case, do not need correct. ++ // Normal case, do not need to correct. + return false; + } + } +@@ -2741,6 +2763,18 @@ public class DefaultMessageStore implements MessageStore { + + } + ++ @Override ++ public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { ++ if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() ++ && DefaultMessageStore.this.messageArrivingListener != null) { ++ DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), ++ dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, ++ dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), ++ dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); ++ DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest); ++ } ++ } ++ + class ReputMessageService extends ServiceThread { + + protected volatile long reputFromOffset = 0; +@@ -2810,13 +2844,8 @@ public class DefaultMessageStore implements MessageStore { + if (size > 0) { + DefaultMessageStore.this.doDispatch(dispatchRequest); + +- if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() +- && DefaultMessageStore.this.messageArrivingListener != null) { +- DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), +- dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, +- dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), +- dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); +- notifyMessageArrive4MultiQueue(dispatchRequest); ++ if (!notifyMessageArriveInBatch) { ++ notifyMessageArriveIfNecessary(dispatchRequest); + } + + this.reputFromOffset += size; +@@ -2850,9 +2879,14 @@ public class DefaultMessageStore implements MessageStore { + } + } + } ++ } catch (RocksDBException e) { ++ ERROR_LOG.info("dispatch message to cq exception. reputFromOffset: {}", this.reputFromOffset, e); ++ return; + } finally { + result.release(); + } ++ ++ finishCommitLogDispatch(); + } + } + +@@ -2989,7 +3023,7 @@ public class DefaultMessageStore implements MessageStore { + // dispatchRequestsList:[ + // {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}, + // {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}] +- private void dispatch() { ++ private void dispatch() throws Exception { + dispatchRequestsList.clear(); + dispatchRequestOrderlyQueue.get(dispatchRequestsList); + if (!dispatchRequestsList.isEmpty()) { +@@ -2997,21 +3031,15 @@ public class DefaultMessageStore implements MessageStore { + for (DispatchRequest dispatchRequest : dispatchRequests) { + DefaultMessageStore.this.doDispatch(dispatchRequest); + // wake up long-polling +- if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() +- && DefaultMessageStore.this.messageArrivingListener != null) { +- DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), +- dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, +- dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), +- dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); +- DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest); +- } ++ DefaultMessageStore.this.notifyMessageArriveIfNecessary(dispatchRequest); ++ + if (!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() && +- DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) { ++ DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) { + DefaultMessageStore.this.storeStatsService +- .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1); ++ .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1); + DefaultMessageStore.this.storeStatsService +- .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()) +- .add(dispatchRequest.getMsgSize()); ++ .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()) ++ .add(dispatchRequest.getMsgSize()); + } + } + } +@@ -3079,7 +3107,7 @@ public class DefaultMessageStore implements MessageStore { + public void doReput() { + if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) { + LOGGER.warn("The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.", +- this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); ++ this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); + this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset(); + } + for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) { +@@ -3138,6 +3166,9 @@ public class DefaultMessageStore implements MessageStore { + result.release(); + } + } ++ ++ // only for rocksdb mode ++ finishCommitLogDispatch(); + } + + /** +@@ -3180,8 +3211,8 @@ public class DefaultMessageStore implements MessageStore { + + if (this.isCommitLogAvailable()) { + LOGGER.warn("shutdown concurrentReputMessageService, but CommitLog have not finish to be dispatched, CommitLog max" + +- " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(), +- this.reputFromOffset); ++ " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(), ++ this.reputFromOffset); + } + + this.mainBatchDispatchRequestService.shutdown(); +diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +index 989cbbe31..814c6d1bf 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +@@ -16,10 +16,6 @@ + */ + package org.apache.rocketmq.store; + +-import io.opentelemetry.api.common.AttributesBuilder; +-import io.opentelemetry.api.metrics.Meter; +-import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.nio.ByteBuffer; + import java.util.HashMap; + import java.util.LinkedList; +@@ -40,10 +36,15 @@ import org.apache.rocketmq.store.hook.PutMessageHook; + import org.apache.rocketmq.store.hook.SendMessageBackHook; + import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +-import org.apache.rocketmq.store.queue.ConsumeQueueStore; ++import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; + import org.apache.rocketmq.store.stats.BrokerStatsManager; + import org.apache.rocketmq.store.timer.TimerMessageStore; + import org.apache.rocketmq.store.util.PerfCounter; ++import org.rocksdb.RocksDBException; ++import io.opentelemetry.api.common.AttributesBuilder; ++import io.opentelemetry.api.metrics.Meter; ++import io.opentelemetry.sdk.metrics.InstrumentSelector; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + + /** + * This class defines contracting interfaces to implement, allowing third-party vendor to use customized message store. +@@ -545,7 +546,7 @@ public interface MessageStore { + void setConfirmOffset(long phyOffset); + + /** +- * Check if the operation system page cache is busy or not. ++ * Check if the operating system page cache is busy or not. + * + * @return true if the OS page cache is busy; false otherwise. + */ +@@ -620,9 +621,18 @@ public interface MessageStore { + * @param commitLogFile commit log file + * @param isRecover is from recover process + * @param isFileEnd if the dispatch request represents 'file end' ++ * @throws RocksDBException only in rocksdb mode + */ + void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, +- boolean isRecover, boolean isFileEnd); ++ boolean isRecover, boolean isFileEnd) throws RocksDBException; ++ ++ /** ++ * Only used in rocksdb mode, because we build consumeQueue in batch(default 16 dispatchRequests) ++ * It will be triggered in two cases: ++ * @see org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput ++ * @see CommitLog#recoverAbnormally ++ */ ++ void finishCommitLogDispatch(); + + /** + * Get the message store config +@@ -691,13 +701,9 @@ public interface MessageStore { + * Truncate dirty logic files + * + * @param phyOffset physical offset ++ * @throws RocksDBException only in rocksdb mode + */ +- void truncateDirtyLogicFiles(long phyOffset); +- +- /** +- * Destroy logics files +- */ +- void destroyLogics(); ++ void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException; + + /** + * Unlock mappedFile +@@ -718,7 +724,7 @@ public interface MessageStore { + * + * @return the queue store + */ +- ConsumeQueueStore getQueueStore(); ++ ConsumeQueueStoreInterface getQueueStore(); + + /** + * If 'sync disk flush' is configured in this message store +@@ -739,8 +745,9 @@ public interface MessageStore { + * yourself. + * + * @param msg message ++ * @throws RocksDBException + */ +- void assignOffset(MessageExtBrokerInner msg); ++ void assignOffset(MessageExtBrokerInner msg) throws RocksDBException; + + /** + * Increase queue offset in memory table. If there is a race condition, you need to lock/unlock this method +@@ -835,14 +842,15 @@ public interface MessageStore { + * + * @param offsetToTruncate offset to truncate + * @return true if truncate succeed, false otherwise ++ * @throws RocksDBException only in rocksdb mode + */ +- boolean truncateFiles(long offsetToTruncate); ++ boolean truncateFiles(long offsetToTruncate) throws RocksDBException; + + /** +- * Check if the offset is align with one message. ++ * Check if the offset is aligned with one message. + * + * @param offset offset to check +- * @return true if align, false otherwise ++ * @return true if aligned, false otherwise + */ + boolean isOffsetAligned(long offset); + +@@ -971,4 +979,14 @@ public interface MessageStore { + * @param attributesBuilderSupplier metrics attributes builder + */ + void initMetrics(Meter meter, Supplier attributesBuilderSupplier); ++ ++ /** ++ * Recover topic queue table ++ */ ++ void recoverTopicQueueTable(); ++ ++ /** ++ * notify message arrive if necessary ++ */ ++ void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest); + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +new file mode 100644 +index 000000000..87ccb5474 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +@@ -0,0 +1,169 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store; ++ ++import java.io.IOException; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++ ++import org.apache.rocketmq.common.BrokerConfig; ++import org.apache.rocketmq.common.TopicConfig; ++import org.apache.rocketmq.common.UtilAll; ++import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.apache.rocketmq.store.config.StorePathConfigHelper; ++import org.apache.rocketmq.store.queue.ConsumeQueueInterface; ++import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; ++import org.apache.rocketmq.store.queue.RocksDBConsumeQueue; ++import org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore; ++import org.apache.rocketmq.store.stats.BrokerStatsManager; ++import org.rocksdb.RocksDBException; ++ ++public class RocksDBMessageStore extends DefaultMessageStore { ++ ++ public RocksDBMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, ++ final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws ++ IOException { ++ super(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, topicConfigTable); ++ notifyMessageArriveInBatch = true; ++ } ++ ++ @Override ++ public ConsumeQueueStoreInterface createConsumeQueueStore() { ++ return new RocksDBConsumeQueueStore(this); ++ } ++ ++ @Override ++ public CleanConsumeQueueService createCleanConsumeQueueService() { ++ return new RocksDBCleanConsumeQueueService(); ++ } ++ ++ @Override ++ public FlushConsumeQueueService createFlushConsumeQueueService() { ++ return new RocksDBFlushConsumeQueueService(); ++ } ++ ++ @Override ++ public CorrectLogicOffsetService createCorrectLogicOffsetService() { ++ return new RocksDBCorrectLogicOffsetService(); ++ } ++ ++ /** ++ * Try to set topicQueueTable = new HashMap<>(), otherwise it will cause bug when broker role changes. ++ * And unlike method in DefaultMessageStore, we don't need to really recover topic queue table advance, ++ * because we can recover topic queue table from rocksdb when we need to use it. ++ * @see RocksDBConsumeQueue#assignQueueOffset ++ */ ++ @Override ++ public void recoverTopicQueueTable() { ++ this.consumeQueueStore.setTopicQueueTable(new ConcurrentHashMap<>()); ++ } ++ ++ @Override ++ public void finishCommitLogDispatch() { ++ try { ++ putMessagePositionInfo(null); ++ } catch (RocksDBException e) { ++ ERROR_LOG.info("try to finish commitlog dispatch error.", e); ++ } ++ } ++ ++ @Override ++ public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) { ++ return findConsumeQueue(topic, queueId); ++ } ++ ++ class RocksDBCleanConsumeQueueService extends CleanConsumeQueueService { ++ private final double diskSpaceWarningLevelRatio = ++ Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceWarningLevelRatio", "0.90")); ++ ++ private final double diskSpaceCleanForciblyRatio = ++ Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceCleanForciblyRatio", "0.85")); ++ ++ @Override ++ protected void deleteExpiredFiles() { ++ ++ long minOffset = RocksDBMessageStore.this.commitLog.getMinOffset(); ++ if (minOffset > this.lastPhysicalMinOffset) { ++ this.lastPhysicalMinOffset = minOffset; ++ ++ boolean spaceFull = isSpaceToDelete(); ++ boolean timeUp = cleanCommitLogService.isTimeToDelete(); ++ if (spaceFull || timeUp) { ++ RocksDBMessageStore.this.consumeQueueStore.cleanExpired(minOffset); ++ } ++ ++ RocksDBMessageStore.this.indexService.deleteExpiredFile(minOffset); ++ } ++ } ++ ++ private boolean isSpaceToDelete() { ++ double ratio = RocksDBMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0; ++ ++ String storePathLogics = StorePathConfigHelper ++ .getStorePathConsumeQueue(RocksDBMessageStore.this.getMessageStoreConfig().getStorePathRootDir()); ++ double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics); ++ if (logicsRatio > diskSpaceWarningLevelRatio) { ++ boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskFull(); ++ if (diskOk) { ++ RocksDBMessageStore.LOGGER.error("logics disk maybe full soon " + logicsRatio + ", so mark disk full"); ++ } ++ } else if (logicsRatio > diskSpaceCleanForciblyRatio) { ++ } else { ++ boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskOK(); ++ if (!diskOk) { ++ RocksDBMessageStore.LOGGER.info("logics disk space OK " + logicsRatio + ", so mark disk ok"); ++ } ++ } ++ ++ if (logicsRatio < 0 || logicsRatio > ratio) { ++ RocksDBMessageStore.LOGGER.info("logics disk maybe full soon, so reclaim space, " + logicsRatio); ++ return true; ++ } ++ ++ return false; ++ } ++ } ++ ++ class RocksDBFlushConsumeQueueService extends FlushConsumeQueueService { ++ /** ++ * There is no need to flush consume queue, ++ * we put all consume queues in RocksDBConsumeQueueStore, ++ * it depends on rocksdb to flush consume queue to disk(sorted string table), ++ * we even don't flush WAL of consume store, since we think it can recover consume queue from commitlog. ++ */ ++ @Override ++ public void run() { ++ ++ } ++ } ++ ++ class RocksDBCorrectLogicOffsetService extends CorrectLogicOffsetService { ++ /** ++ * There is no need to correct min offset of consume queue, we already fix this problem. ++ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset ++ */ ++ public void run() { ++ ++ } ++ } ++ ++ @Override ++ public long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter) { ++ // todo ++ return 0; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java +index 2ae6879aa..91fcb155a 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java ++++ b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java +@@ -30,6 +30,8 @@ public class RunningFlags { + + private static final int FENCED_BIT = 1 << 5; + ++ private static final int LOGIC_DISK_FULL_BIT = 1 << 5; ++ + private volatile int flagBits = 0; + + public RunningFlags() { +@@ -63,6 +65,10 @@ public class RunningFlags { + return result; + } + ++ public void clearLogicsQueueError() { ++ this.flagBits &= ~WRITE_LOGICS_QUEUE_ERROR_BIT; ++ } ++ + public boolean getAndMakeWriteable() { + boolean result = this.isWriteable(); + if (!result) { +@@ -72,7 +78,7 @@ public class RunningFlags { + } + + public boolean isWriteable() { +- if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT)) == 0) { ++ if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT | LOGIC_DISK_FULL_BIT)) == 0) { + return true; + } + +@@ -81,7 +87,7 @@ public class RunningFlags { + + //for consume queue, just ignore the DISK_FULL_BIT + public boolean isCQWriteable() { +- if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT)) == 0) { ++ if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT | LOGIC_DISK_FULL_BIT)) == 0) { + return true; + } + +@@ -139,4 +145,16 @@ public class RunningFlags { + this.flagBits &= ~DISK_FULL_BIT; + return result; + } ++ ++ public boolean getAndMakeLogicDiskFull() { ++ boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT); ++ this.flagBits |= LOGIC_DISK_FULL_BIT; ++ return result; ++ } ++ ++ public boolean getAndMakeLogicDiskOK() { ++ boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT); ++ this.flagBits &= ~LOGIC_DISK_FULL_BIT; ++ return result; ++ } + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +index 9fa448043..028facbdc 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java ++++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +@@ -397,8 +397,10 @@ public class MessageStoreConfig { + private int batchDispatchRequestThreadPoolNums = 16; + + // rocksdb mode ++ private long cleanRocksDBDirtyCQIntervalMin = 60; ++ private long statRocksDBCQIntervalSec = 10; ++ private long memTableFlushIntervalMs = 60 * 60 * 1000L; + private boolean realTimePersistRocksDBConfig = true; +- private long memTableFlushInterval = 60 * 60 * 1000L; + private boolean enableRocksDBLog = false; + + private int topicQueueLockNum = 32; +@@ -499,6 +501,10 @@ public class MessageStoreConfig { + this.mappedFileSizeCommitLog = mappedFileSizeCommitLog; + } + ++ public boolean isEnableRocksDBStore() { ++ return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.storeType); ++ } ++ + public String getStoreType() { + return storeType; + } +@@ -508,7 +514,6 @@ public class MessageStoreConfig { + } + + public int getMappedFileSizeConsumeQueue() { +- + int factor = (int) Math.ceil(this.mappedFileSizeConsumeQueue / (ConsumeQueue.CQ_STORE_UNIT_SIZE * 1.0)); + return (int) (factor * ConsumeQueue.CQ_STORE_UNIT_SIZE); + } +@@ -1738,12 +1743,28 @@ public class MessageStoreConfig { + this.realTimePersistRocksDBConfig = realTimePersistRocksDBConfig; + } + +- public long getMemTableFlushInterval() { +- return memTableFlushInterval; ++ public long getStatRocksDBCQIntervalSec() { ++ return statRocksDBCQIntervalSec; ++ } ++ ++ public void setStatRocksDBCQIntervalSec(long statRocksDBCQIntervalSec) { ++ this.statRocksDBCQIntervalSec = statRocksDBCQIntervalSec; ++ } ++ ++ public long getCleanRocksDBDirtyCQIntervalMin() { ++ return cleanRocksDBDirtyCQIntervalMin; ++ } ++ ++ public void setCleanRocksDBDirtyCQIntervalMin(long cleanRocksDBDirtyCQIntervalMin) { ++ this.cleanRocksDBDirtyCQIntervalMin = cleanRocksDBDirtyCQIntervalMin; ++ } ++ ++ public long getMemTableFlushIntervalMs() { ++ return memTableFlushIntervalMs; + } + +- public void setMemTableFlushInterval(long memTableFlushInterval) { +- this.memTableFlushInterval = memTableFlushInterval; ++ public void setMemTableFlushIntervalMs(long memTableFlushIntervalMs) { ++ this.memTableFlushIntervalMs = memTableFlushIntervalMs; + } + + public boolean isEnableRocksDBLog() { +diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +index d5f6acdc0..70371d83b 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java ++++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +@@ -16,26 +16,13 @@ + */ + package org.apache.rocketmq.store.dledger; + +-import io.openmessaging.storage.dledger.AppendFuture; +-import io.openmessaging.storage.dledger.BatchAppendFuture; +-import io.openmessaging.storage.dledger.DLedgerConfig; +-import io.openmessaging.storage.dledger.DLedgerServer; +-import io.openmessaging.storage.dledger.entry.DLedgerEntry; +-import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; +-import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; +-import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest; +-import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; +-import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; +-import io.openmessaging.storage.dledger.store.file.MmapFile; +-import io.openmessaging.storage.dledger.store.file.MmapFileList; +-import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; +-import io.openmessaging.storage.dledger.utils.DLedgerUtils; + import java.net.Inet6Address; + import java.net.InetSocketAddress; + import java.nio.ByteBuffer; + import java.util.LinkedList; + import java.util.List; + import java.util.concurrent.CompletableFuture; ++ + import org.apache.rocketmq.common.UtilAll; + import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExtBatch; +@@ -54,6 +41,22 @@ import org.apache.rocketmq.store.SelectMappedBufferResult; + import org.apache.rocketmq.store.StoreStatsService; + import org.apache.rocketmq.store.config.MessageStoreConfig; + import org.apache.rocketmq.store.logfile.MappedFile; ++import org.rocksdb.RocksDBException; ++ ++import io.openmessaging.storage.dledger.AppendFuture; ++import io.openmessaging.storage.dledger.BatchAppendFuture; ++import io.openmessaging.storage.dledger.DLedgerConfig; ++import io.openmessaging.storage.dledger.DLedgerServer; ++import io.openmessaging.storage.dledger.entry.DLedgerEntry; ++import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; ++import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; ++import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest; ++import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; ++import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; ++import io.openmessaging.storage.dledger.store.file.MmapFile; ++import io.openmessaging.storage.dledger.store.file.MmapFileList; ++import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; ++import io.openmessaging.storage.dledger.utils.DLedgerUtils; + + /** + * Store all metadata downtime for recovery, data protection reliability +@@ -269,7 +272,7 @@ public class DLedgerCommitLog extends CommitLog { + + return null; + } +- ++ + @Override + public boolean getData(final long offset, final int size, final ByteBuffer byteBuffer) { + if (offset < dividedCommitlogOffset) { +@@ -287,7 +290,7 @@ public class DLedgerCommitLog extends CommitLog { + return false; + } + +- private void recover(long maxPhyOffsetOfConsumeQueue) { ++ private void recover(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + dLedgerFileStore.load(); + if (dLedgerFileList.getMappedFiles().size() > 0) { + dLedgerFileStore.recover(); +@@ -341,12 +344,12 @@ public class DLedgerCommitLog extends CommitLog { + } + + @Override +- public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { ++ public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + recover(maxPhyOffsetOfConsumeQueue); + } + + @Override +- public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { ++ public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + recover(maxPhyOffsetOfConsumeQueue); + } + +@@ -469,9 +472,6 @@ public class DLedgerCommitLog extends CommitLog { + String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); + elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.getData().length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock); +- } catch (Exception e) { +- log.error("Put message error", e); +- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { + beginTimeInDledgerLock = 0; + putMessageLock.unlock(); +@@ -482,6 +482,9 @@ public class DLedgerCommitLog extends CommitLog { + } + + defaultMessageStore.increaseOffset(msg, getMessageNum(msg)); ++ } catch (Exception e) { ++ log.error("Put message error", e); ++ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { + topicQueueLock.unlock(topicQueueKey); + } +@@ -611,9 +614,6 @@ public class DLedgerCommitLog extends CommitLog { + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen, + msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock); + appendResult.setMsgNum(msgNum); +- } catch (Exception e) { +- log.error("Put message error", e); +- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { + beginTimeInDledgerLock = 0; + putMessageLock.unlock(); +@@ -626,7 +626,10 @@ public class DLedgerCommitLog extends CommitLog { + + defaultMessageStore.increaseOffset(messageExtBatch, (short) batchNum); + +- } finally { ++ } catch (Exception e) { ++ log.error("Put message error", e); ++ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); ++ } finally { + topicQueueLock.unlock(encodeResult.queueOffsetKey); + } + +diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +index 467da603d..aaea7d690 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +@@ -25,6 +25,7 @@ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; + import org.apache.rocketmq.store.CommitLog; + import org.apache.rocketmq.store.DefaultMessageStore; + import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.rocksdb.RocksDBException; + + public interface HAService { + +@@ -53,7 +54,7 @@ public interface HAService { + * + * @param masterEpoch the new masterEpoch + */ +- default boolean changeToMaster(int masterEpoch) { ++ default boolean changeToMaster(int masterEpoch) throws RocksDBException { + return false; + } + +diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java +index 936db0c4c..176c25a96 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java +@@ -432,7 +432,7 @@ public class AutoSwitchHAClient extends ServiceThread implements HAClient { + /** + * Compare the master and slave's epoch file, find consistent point, do truncate. + */ +- private boolean doTruncate(List masterEpochEntries, long masterEndOffset) throws IOException { ++ private boolean doTruncate(List masterEpochEntries, long masterEndOffset) throws Exception { + if (this.epochCache.getEntrySize() == 0) { + // If epochMap is empty, means the broker is a new replicas + LOGGER.info("Slave local epochCache is empty, skip truncate log"); +diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +index f20bc3e28..64dad9aef 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +@@ -51,6 +51,7 @@ import org.apache.rocketmq.store.ha.GroupTransferService; + import org.apache.rocketmq.store.ha.HAClient; + import org.apache.rocketmq.store.ha.HAConnection; + import org.apache.rocketmq.store.ha.HAConnectionStateNotificationService; ++import org.rocksdb.RocksDBException; + + /** + * SwitchAble ha service, support switch role to master or slave. +@@ -111,7 +112,7 @@ public class AutoSwitchHAService extends DefaultHAService { + } + + @Override +- public boolean changeToMaster(int masterEpoch) { ++ public boolean changeToMaster(int masterEpoch) throws RocksDBException { + final int lastEpoch = this.epochCache.lastEpoch(); + if (masterEpoch < lastEpoch) { + LOGGER.warn("newMasterEpoch {} < lastEpoch {}, fail to change to master", masterEpoch, lastEpoch); +@@ -315,7 +316,7 @@ public class AutoSwitchHAService extends DefaultHAService { + final EpochEntry currentLeaderEpoch = this.epochCache.lastEntry(); + if (slaveMaxOffset >= currentLeaderEpoch.getStartOffset()) { + LOGGER.info("The slave {} has caught up, slaveMaxOffset: {}, confirmOffset: {}, epoch: {}, leader epoch startOffset: {}.", +- slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset()); ++ slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset()); + currentSyncStateSet.add(slaveBrokerId); + markSynchronizingSyncStateSet(currentSyncStateSet); + // Notify the upper layer that syncStateSet changed. +@@ -491,7 +492,7 @@ public class AutoSwitchHAService extends DefaultHAService { + /** + * Try to truncate incomplete msg transferred from master. + */ +- public long truncateInvalidMsg() { ++ public long truncateInvalidMsg() throws RocksDBException { + long dispatchBehind = this.defaultMessageStore.dispatchBehindBytes(); + if (dispatchBehind <= 0) { + LOGGER.info("Dispatch complete, skip truncate"); +diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +index ab9fc6da7..2f2ce9812 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +@@ -17,10 +17,6 @@ + + package org.apache.rocketmq.store.plugin; + +-import io.opentelemetry.api.common.AttributesBuilder; +-import io.opentelemetry.api.metrics.Meter; +-import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.nio.ByteBuffer; + import java.util.HashMap; + import java.util.LinkedList; +@@ -55,10 +51,16 @@ import org.apache.rocketmq.store.hook.PutMessageHook; + import org.apache.rocketmq.store.hook.SendMessageBackHook; + import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +-import org.apache.rocketmq.store.queue.ConsumeQueueStore; ++import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; + import org.apache.rocketmq.store.stats.BrokerStatsManager; + import org.apache.rocketmq.store.timer.TimerMessageStore; + import org.apache.rocketmq.store.util.PerfCounter; ++import org.rocksdb.RocksDBException; ++ ++import io.opentelemetry.api.common.AttributesBuilder; ++import io.opentelemetry.api.metrics.Meter; ++import io.opentelemetry.sdk.metrics.InstrumentSelector; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + + public abstract class AbstractPluginMessageStore implements MessageStore { + protected MessageStore next = null; +@@ -457,7 +459,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + } + + @Override +- public boolean truncateFiles(long offsetToTruncate) { ++ public boolean truncateFiles(long offsetToTruncate) throws RocksDBException { + return next.truncateFiles(offsetToTruncate); + } + +@@ -511,7 +513,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + + @Override + public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, +- boolean isRecover, boolean isFileEnd) { ++ boolean isRecover, boolean isFileEnd) throws RocksDBException { + next.onCommitLogDispatch(dispatchRequest, doDispatch, commitLogFile, isRecover, isFileEnd); + } + +@@ -551,15 +553,10 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + } + + @Override +- public void truncateDirtyLogicFiles(long phyOffset) { ++ public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException { + next.truncateDirtyLogicFiles(phyOffset); + } + +- @Override +- public void destroyLogics() { +- next.destroyLogics(); +- } +- + @Override + public void unlockMappedFile(MappedFile unlockMappedFile) { + next.unlockMappedFile(unlockMappedFile); +@@ -571,7 +568,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + } + + @Override +- public ConsumeQueueStore getQueueStore() { ++ public ConsumeQueueStoreInterface getQueueStore() { + return next.getQueueStore(); + } + +@@ -586,7 +583,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + } + + @Override +- public void assignOffset(MessageExtBrokerInner msg) { ++ public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException { + next.assignOffset(msg); + } + +@@ -649,4 +646,19 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + public void initMetrics(Meter meter, Supplier attributesBuilderSupplier) { + next.initMetrics(meter, attributesBuilderSupplier); + } ++ ++ @Override ++ public void finishCommitLogDispatch() { ++ next.finishCommitLogDispatch(); ++ } ++ ++ @Override ++ public void recoverTopicQueueTable() { ++ next.recoverTopicQueueTable(); ++ } ++ ++ @Override ++ public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { ++ next.notifyMessageArriveIfNecessary(dispatchRequest); ++ } + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java +new file mode 100644 +index 000000000..30054fa50 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java +@@ -0,0 +1,105 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.DefaultMessageStore; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.rocksdb.RocksDBException; ++ ++public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface { ++ protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ ++ protected final DefaultMessageStore messageStore; ++ protected final MessageStoreConfig messageStoreConfig; ++ protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator(); ++ protected final ConcurrentMap> consumeQueueTable; ++ ++ public AbstractConsumeQueueStore(DefaultMessageStore messageStore) { ++ this.messageStore = messageStore; ++ this.messageStoreConfig = messageStore.getMessageStoreConfig(); ++ this.consumeQueueTable = new ConcurrentHashMap<>(32); ++ } ++ ++ @Override ++ public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) { ++ consumeQueue.putMessagePositionInfoWrapper(request); ++ } ++ ++ @Override ++ public Long getMaxOffset(String topic, int queueId) { ++ return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); ++ } ++ ++ @Override ++ public void setTopicQueueTable(ConcurrentMap topicQueueTable) { ++ this.queueOffsetOperator.setTopicQueueTable(topicQueueTable); ++ this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable); ++ } ++ ++ @Override ++ public ConcurrentMap getTopicQueueTable() { ++ return this.queueOffsetOperator.getTopicQueueTable(); ++ } ++ ++ @Override ++ public void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException { ++ ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); ++ consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg); ++ } ++ ++ @Override ++ public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) { ++ ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); ++ consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum); ++ } ++ ++ @Override ++ public void removeTopicQueueTable(String topic, Integer queueId) { ++ this.queueOffsetOperator.remove(topic, queueId); ++ } ++ ++ @Override ++ public ConcurrentMap> getConsumeQueueTable() { ++ return this.consumeQueueTable; ++ } ++ ++ @Override ++ public ConcurrentMap findConsumeQueueMap(String topic) { ++ return this.consumeQueueTable.get(topic); ++ } ++ ++ @Override ++ public long getStoreTime(CqUnit cqUnit) { ++ if (cqUnit != null) { ++ try { ++ final long phyOffset = cqUnit.getPos(); ++ final int size = cqUnit.getSize(); ++ long storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size); ++ return storeTime; ++ } catch (Exception e) { ++ } ++ } ++ return -1; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java +index 387c233bf..7108c835c 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java +@@ -24,6 +24,7 @@ import java.util.Map; + import java.util.concurrent.ConcurrentSkipListMap; + import java.util.function.Function; + import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.attribute.CQType; + import org.apache.rocketmq.common.constant.LoggerName; +@@ -311,6 +312,11 @@ public class BatchConsumeQueue implements ConsumeQueueInterface { + return new BatchConsumeQueueIterator(sbr); + } + ++ @Override ++ public ReferredIterator iterateFrom(long startIndex, int count) { ++ return iterateFrom(startIndex); ++ } ++ + @Override + public CqUnit get(long offset) { + ReferredIterator it = iterateFrom(offset); +@@ -320,6 +326,20 @@ public class BatchConsumeQueue implements ConsumeQueueInterface { + return it.nextAndRelease(); + } + ++ @Override ++ public Pair getCqUnitAndStoreTime(long index) { ++ CqUnit cqUnit = get(index); ++ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); ++ return new Pair<>(cqUnit, messageStoreTime); ++ } ++ ++ @Override ++ public Pair getEarliestUnitAndStoreTime() { ++ CqUnit cqUnit = getEarliestUnit(); ++ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); ++ return new Pair<>(cqUnit, messageStoreTime); ++ } ++ + @Override + public CqUnit getEarliestUnit() { + return get(minOffsetInQueue); +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java +index 55d080829..c65f2a68b 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java +@@ -18,10 +18,12 @@ + package org.apache.rocketmq.store.queue; + + import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.attribute.CQType; + import org.apache.rocketmq.common.message.MessageExtBrokerInner; + import org.apache.rocketmq.store.DispatchRequest; + import org.apache.rocketmq.store.MessageFilter; ++import org.rocksdb.RocksDBException; + + public interface ConsumeQueueInterface extends FileQueueLifeCycle { + /** +@@ -44,6 +46,16 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { + */ + ReferredIterator iterateFrom(long startIndex); + ++ /** ++ * Get the units from the start offset. ++ * ++ * @param startIndex start index ++ * @param count the unit counts will be iterated ++ * @return the unit iterateFrom ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ ReferredIterator iterateFrom(long startIndex, int count) throws RocksDBException; ++ + /** + * Get cq unit at specified index + * @param index index +@@ -51,6 +63,18 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { + */ + CqUnit get(long index); + ++ /** ++ * Get earliest cq unit ++ * @return the cq unit and message storeTime at index ++ */ ++ Pair getCqUnitAndStoreTime(long index); ++ ++ /** ++ * Get earliest cq unit ++ * @return earliest cq unit and message storeTime ++ */ ++ Pair getEarliestUnitAndStoreTime(); ++ + /** + * Get earliest cq unit + * @return earliest cq unit +@@ -153,8 +177,9 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { + * Assign queue offset. + * @param queueOffsetAssigner the delegated queue offset assigner + * @param msg message itself ++ * @throws RocksDBException only in rocksdb mode + */ +- void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg); ++ void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg) throws RocksDBException; + + + /** +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +index d03d15d65..616511b67 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +@@ -16,64 +16,128 @@ + */ + package org.apache.rocketmq.store.queue; + ++import java.io.File; + import java.nio.ByteBuffer; + import java.util.ArrayList; ++import java.util.Iterator; + import java.util.List; ++import java.util.Map; ++import java.util.Map.Entry; ++import java.util.Objects; ++import java.util.Optional; + import java.util.concurrent.BlockingQueue; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.CountDownLatch; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.FutureTask; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.TimeUnit; ++import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.TopicConfig; + import org.apache.rocketmq.common.attribute.CQType; +-import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.common.utils.QueueTypeUtils; + import org.apache.rocketmq.common.utils.ThreadUtils; +-import org.apache.rocketmq.logging.org.slf4j.Logger; +-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.store.CommitLog; + import org.apache.rocketmq.store.ConsumeQueue; + import org.apache.rocketmq.store.DefaultMessageStore; + import org.apache.rocketmq.store.DispatchRequest; +-import org.apache.rocketmq.common.message.MessageExtBrokerInner; + import org.apache.rocketmq.store.SelectMappedBufferResult; +-import org.apache.rocketmq.store.config.MessageStoreConfig; +- +-import java.io.File; +-import java.util.Iterator; +-import java.util.Map; +-import java.util.Objects; +-import java.util.Optional; +-import java.util.concurrent.ConcurrentHashMap; +-import java.util.concurrent.ConcurrentMap; + + import static java.lang.String.format; + import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue; + import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue; + +-public class ConsumeQueueStore { +- private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++public class ConsumeQueueStore extends AbstractConsumeQueueStore { + +- protected final DefaultMessageStore messageStore; +- protected final MessageStoreConfig messageStoreConfig; +- protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator(); +- protected final ConcurrentMap> consumeQueueTable; ++ public ConsumeQueueStore(DefaultMessageStore messageStore) { ++ super(messageStore); ++ } + +- public ConsumeQueueStore(DefaultMessageStore messageStore, MessageStoreConfig messageStoreConfig) { +- this.messageStore = messageStore; +- this.messageStoreConfig = messageStoreConfig; +- this.consumeQueueTable = new ConcurrentHashMap<>(32); ++ @Override ++ public void start() { ++ log.info("Default ConsumeQueueStore start!"); + } + +- private FileQueueLifeCycle getLifeCycle(String topic, int queueId) { +- return findOrCreateConsumeQueue(topic, queueId); ++ @Override ++ public boolean load() { ++ boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ); ++ boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ); ++ return cqLoadResult && bcqLoadResult; ++ } ++ ++ @Override ++ public boolean loadAfterDestroy() { ++ return true; ++ } ++ ++ @Override ++ public void recover() { ++ for (ConcurrentMap maps : this.consumeQueueTable.values()) { ++ for (ConsumeQueueInterface logic : maps.values()) { ++ this.recover(logic); ++ } ++ } + } + ++ @Override ++ public boolean recoverConcurrently() { ++ int count = 0; ++ for (ConcurrentMap maps : this.consumeQueueTable.values()) { ++ count += maps.values().size(); ++ } ++ final CountDownLatch countDownLatch = new CountDownLatch(count); ++ BlockingQueue recoverQueue = new LinkedBlockingQueue<>(); ++ final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_"); ++ List> result = new ArrayList<>(count); ++ try { ++ for (ConcurrentMap maps : this.consumeQueueTable.values()) { ++ for (final ConsumeQueueInterface logic : maps.values()) { ++ FutureTask futureTask = new FutureTask<>(() -> { ++ boolean ret = true; ++ try { ++ logic.recover(); ++ } catch (Throwable e) { ++ ret = false; ++ log.error("Exception occurs while recover consume queue concurrently, " + ++ "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e); ++ } finally { ++ countDownLatch.countDown(); ++ } ++ return ret; ++ }); ++ ++ result.add(futureTask); ++ executor.submit(futureTask); ++ } ++ } ++ countDownLatch.await(); ++ for (FutureTask task : result) { ++ if (task != null && task.isDone()) { ++ if (!task.get()) { ++ return false; ++ } ++ } ++ } ++ } catch (Exception e) { ++ log.error("Exception occurs while recover consume queue concurrently", e); ++ return false; ++ } finally { ++ executor.shutdown(); ++ } ++ return true; ++ } ++ ++ @Override ++ public boolean shutdown() { ++ return true; ++ } ++ ++ @Override + public long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.rollNextFile(offset); +@@ -83,32 +147,53 @@ public class ConsumeQueueStore { + consumeQueue.correctMinOffset(minCommitLogOffset); + } + +- /** +- * Apply the dispatched request and build the consume queue. This function should be idempotent. +- * +- * @param consumeQueue consume queue +- * @param request dispatch request +- */ +- public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) { +- consumeQueue.putMessagePositionInfoWrapper(request); +- } +- ++ @Override + public void putMessagePositionInfoWrapper(DispatchRequest dispatchRequest) { + ConsumeQueueInterface cq = this.findOrCreateConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId()); + this.putMessagePositionInfoWrapper(cq, dispatchRequest); + } + ++ @Override ++ public List rangeQuery(String topic, int queueId, long startIndex, int num) { ++ return null; ++ } ++ ++ @Override ++ public ByteBuffer get(String topic, int queueId, long startIndex) { ++ return null; ++ } ++ ++ @Override ++ public long getMaxOffsetInQueue(String topic, int queueId) { ++ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); ++ if (logic != null) { ++ return logic.getMaxOffsetInQueue(); ++ } ++ return 0; ++ } ++ ++ @Override ++ public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) { ++ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); ++ if (logic != null) { ++ long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType); ++ // Make sure the result offset is in valid range. ++ resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue()); ++ resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue()); ++ return resultOffset; ++ } ++ return 0; ++ } ++ ++ private FileQueueLifeCycle getLifeCycle(String topic, int queueId) { ++ return findOrCreateConsumeQueue(topic, queueId); ++ } ++ + public boolean load(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.load(); + } + +- public boolean load() { +- boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ); +- boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ); +- return cqLoadResult && bcqLoadResult; +- } +- + private boolean loadConsumeQueues(String storePath, CQType cqType) { + File dirLogic = new File(storePath); + File[] fileTopicList = dirLogic.listFiles(); +@@ -189,62 +274,17 @@ public class ConsumeQueueStore { + fileQueueLifeCycle.recover(); + } + +- public void recover() { +- for (ConcurrentMap maps : this.consumeQueueTable.values()) { +- for (ConsumeQueueInterface logic : maps.values()) { +- this.recover(logic); +- } +- } +- } +- +- public boolean recoverConcurrently() { +- int count = 0; +- for (ConcurrentMap maps : this.consumeQueueTable.values()) { +- count += maps.values().size(); +- } +- final CountDownLatch countDownLatch = new CountDownLatch(count); +- BlockingQueue recoverQueue = new LinkedBlockingQueue<>(); +- final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_"); +- List> result = new ArrayList<>(count); +- try { +- for (ConcurrentMap maps : this.consumeQueueTable.values()) { +- for (final ConsumeQueueInterface logic : maps.values()) { +- FutureTask futureTask = new FutureTask<>(() -> { +- boolean ret = true; +- try { +- logic.recover(); +- } catch (Throwable e) { +- ret = false; +- log.error("Exception occurs while recover consume queue concurrently, " + +- "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e); +- } finally { +- countDownLatch.countDown(); +- } +- return ret; +- }); +- +- result.add(futureTask); +- executor.submit(futureTask); +- } +- } +- countDownLatch.await(); +- for (FutureTask task : result) { +- if (task != null && task.isDone()) { +- if (!task.get()) { +- return false; +- } +- } +- } +- } catch (Exception e) { +- log.error("Exception occurs while recover consume queue concurrently", e); +- return false; +- } finally { +- executor.shutdown(); ++ @Override ++ public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) { ++ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); ++ if (logic != null) { ++ return logic.getMaxPhysicOffset(); + } +- return true; ++ return null; + } + +- public long getMaxOffsetInConsumeQueue() { ++ @Override ++ public long getMaxPhyOffsetInConsumeQueue() { + long maxPhysicOffset = -1L; + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueueInterface logic : maps.values()) { +@@ -256,11 +296,22 @@ public class ConsumeQueueStore { + return maxPhysicOffset; + } + ++ @Override ++ public long getMinOffsetInQueue(String topic, int queueId) { ++ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); ++ if (logic != null) { ++ return logic.getMinOffsetInQueue(); ++ } ++ ++ return -1; ++ } ++ + public void checkSelf(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + fileQueueLifeCycle.checkSelf(); + } + ++ @Override + public void checkSelf() { + for (Map.Entry> topicEntry : this.consumeQueueTable.entrySet()) { + for (Map.Entry cqEntry : topicEntry.getValue().entrySet()) { +@@ -269,16 +320,19 @@ public class ConsumeQueueStore { + } + } + ++ @Override + public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.flush(flushLeastPages); + } + ++ @Override + public void destroy(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + fileQueueLifeCycle.destroy(); + } + ++ @Override + public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.deleteExpiredFile(minCommitLogPos); +@@ -300,21 +354,20 @@ public class ConsumeQueueStore { + fileQueueLifeCycle.cleanSwappedMap(forceCleanSwapIntervalMs); + } + ++ @Override + public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.isFirstFileAvailable(); + } + ++ @Override + public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.isFirstFileExist(); + } + ++ @Override + public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) { +- return doFindOrCreateConsumeQueue(topic, queueId); +- } +- +- private ConsumeQueueInterface doFindOrCreateConsumeQueue(String topic, int queueId) { + ConcurrentMap map = consumeQueueTable.get(topic); + if (null == map) { + ConcurrentMap newMap = new ConcurrentHashMap<>(128); +@@ -361,46 +414,15 @@ public class ConsumeQueueStore { + return logic; + } + +- public Long getMaxOffset(String topic, int queueId) { +- return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); +- } +- +- public void setTopicQueueTable(ConcurrentMap topicQueueTable) { +- this.queueOffsetOperator.setTopicQueueTable(topicQueueTable); +- this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable); +- } +- +- public ConcurrentMap getTopicQueueTable() { +- return this.queueOffsetOperator.getTopicQueueTable(); +- } +- + public void setBatchTopicQueueTable(ConcurrentMap batchTopicQueueTable) { + this.queueOffsetOperator.setBatchTopicQueueTable(batchTopicQueueTable); + } + +- public void assignQueueOffset(MessageExtBrokerInner msg) { +- ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); +- consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg); +- } +- +- public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) { +- ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); +- consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum); +- } +- + public void updateQueueOffset(String topic, int queueId, long offset) { + String topicQueueKey = topic + "-" + queueId; + this.queueOffsetOperator.updateQueueOffset(topicQueueKey, offset); + } + +- public void removeTopicQueueTable(String topic, Integer queueId) { +- this.queueOffsetOperator.remove(topic, queueId); +- } +- +- public ConcurrentMap> getConsumeQueueTable() { +- return consumeQueueTable; +- } +- + private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueueInterface consumeQueue) { + ConcurrentMap map = this.consumeQueueTable.get(topic); + if (null == map) { +@@ -412,6 +434,7 @@ public class ConsumeQueueStore { + } + } + ++ @Override + public void recoverOffsetTable(long minPhyOffset) { + ConcurrentMap cqOffsetTable = new ConcurrentHashMap<>(1024); + ConcurrentMap bcqOffsetTable = new ConcurrentHashMap<>(1024); +@@ -431,7 +454,7 @@ public class ConsumeQueueStore { + } + } + +- //Correct unSubmit consumeOffset ++ // Correct unSubmit consumeOffset + if (messageStoreConfig.isDuplicationEnable()) { + SelectMappedBufferResult lastBuffer = null; + long startReadOffset = messageStore.getCommitLog().getConfirmOffset() == -1 ? 0 : messageStore.getCommitLog().getConfirmOffset(); +@@ -476,6 +499,7 @@ public class ConsumeQueueStore { + this.setBatchTopicQueueTable(bcqOffsetTable); + } + ++ @Override + public void destroy() { + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueueInterface logic : maps.values()) { +@@ -484,8 +508,9 @@ public class ConsumeQueueStore { + } + } + ++ @Override + public void cleanExpired(long minCommitLogOffset) { +- Iterator>> it = this.consumeQueueTable.entrySet().iterator(); ++ Iterator>> it = this.consumeQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry> next = it.next(); + String topic = next.getKey(); +@@ -526,14 +551,16 @@ public class ConsumeQueueStore { + } + } + +- public void truncateDirty(long phyOffset) { ++ @Override ++ public void truncateDirty(long offsetToTruncate) { + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueueInterface logic : maps.values()) { +- this.truncateDirtyLogicFiles(logic, phyOffset); ++ this.truncateDirtyLogicFiles(logic, offsetToTruncate); + } + } + } + ++ @Override + public long getTotalSize() { + long totalSize = 0; + for (ConcurrentMap maps : this.consumeQueueTable.values()) { +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java +new file mode 100644 +index 000000000..268803dcc +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java +@@ -0,0 +1,289 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.nio.ByteBuffer; ++import java.util.List; ++import java.util.concurrent.ConcurrentMap; ++import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.rocksdb.RocksDBException; ++ ++public interface ConsumeQueueStoreInterface { ++ ++ /** ++ * Start the consumeQueueStore ++ */ ++ void start(); ++ ++ /** ++ * Load from file. ++ * @return true if loaded successfully. ++ */ ++ boolean load(); ++ ++ /** ++ * load after destroy ++ */ ++ boolean loadAfterDestroy(); ++ ++ /** ++ * Recover from file. ++ */ ++ void recover(); ++ ++ /** ++ * Recover concurrently from file. ++ * @return true if recovered successfully. ++ */ ++ boolean recoverConcurrently(); ++ ++ /** ++ * Shutdown the consumeQueueStore ++ * @return true if shutdown successfully. ++ */ ++ boolean shutdown(); ++ ++ /** ++ * destroy all consumeQueues ++ */ ++ void destroy(); ++ ++ /** ++ * destroy the specific consumeQueue ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException; ++ ++ /** ++ * Flush cache to file. ++ * @param consumeQueue the consumeQueue will be flushed ++ * @param flushLeastPages the minimum number of pages to be flushed ++ * @return true if any data has been flushed. ++ */ ++ boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages); ++ ++ /** ++ * clean expired data from minPhyOffset ++ * @param minPhyOffset ++ */ ++ void cleanExpired(long minPhyOffset); ++ ++ /** ++ * Check files. ++ */ ++ void checkSelf(); ++ ++ /** ++ * Delete expired files ending at min commit log position. ++ * @param consumeQueue ++ * @param minCommitLogPos min commit log position ++ * @return deleted file numbers. ++ */ ++ int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos); ++ ++ /** ++ * Is the first file available? ++ * @param consumeQueue ++ * @return true if it's available ++ */ ++ boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue); ++ ++ /** ++ * Does the first file exist? ++ * @param consumeQueue ++ * @return true if it exists ++ */ ++ boolean isFirstFileExist(ConsumeQueueInterface consumeQueue); ++ ++ /** ++ * Roll to next file. ++ * @param consumeQueue ++ * @param offset next beginning offset ++ * @return the beginning offset of the next file ++ */ ++ long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset); ++ ++ /** ++ * truncate dirty data ++ * @param offsetToTruncate ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ void truncateDirty(long offsetToTruncate) throws RocksDBException; ++ ++ /** ++ * Apply the dispatched request and build the consume queue. This function should be idempotent. ++ * ++ * @param consumeQueue consume queue ++ * @param request dispatch request ++ */ ++ void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request); ++ ++ /** ++ * Apply the dispatched request. This function should be idempotent. ++ * ++ * @param request dispatch request ++ * @throws RocksDBException only in rocksdb mode will throw exception ++ */ ++ void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException; ++ ++ /** ++ * range query cqUnit(ByteBuffer) in rocksdb ++ * @param topic ++ * @param queueId ++ * @param startIndex ++ * @param num ++ * @return the byteBuffer list of the topic-queueId in rocksdb ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException; ++ ++ /** ++ * query cqUnit(ByteBuffer) in rocksdb ++ * @param topic ++ * @param queueId ++ * @param startIndex ++ * @return the byteBuffer of the topic-queueId in rocksdb ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ ByteBuffer get(final String topic, final int queueId, final long startIndex) throws RocksDBException; ++ ++ /** ++ * get consumeQueue table ++ * @return the consumeQueue table ++ */ ++ ConcurrentMap> getConsumeQueueTable(); ++ ++ /** ++ * Assign queue offset. ++ * @param msg message itself ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException; ++ ++ /** ++ * Increase queue offset. ++ * @param msg message itself ++ * @param messageNum message number ++ */ ++ void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum); ++ ++ /** ++ * recover topicQueue table by minPhyOffset ++ * @param minPhyOffset ++ */ ++ void recoverOffsetTable(long minPhyOffset); ++ ++ /** ++ * set topicQueue table ++ * @param topicQueueTable ++ */ ++ void setTopicQueueTable(ConcurrentMap topicQueueTable); ++ ++ /** ++ * remove topic-queueId from topicQueue table ++ * @param topic ++ * @param queueId ++ */ ++ void removeTopicQueueTable(String topic, Integer queueId); ++ ++ /** ++ * get topicQueue table ++ * @return the topicQueue table ++ */ ++ ConcurrentMap getTopicQueueTable(); ++ ++ /** ++ * get the max physical offset in consumeQueue ++ * @param topic ++ * @param queueId ++ * @return ++ */ ++ Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId); ++ ++ /** ++ * get maxOffset of specific topic-queueId in topicQueue table ++ * @param topic ++ * @param queueId ++ * @return the max offset in QueueOffsetOperator ++ */ ++ Long getMaxOffset(String topic, int queueId); ++ ++ /** ++ * get max physic offset in consumeQueue ++ * @return the max physic offset in consumeQueue ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ long getMaxPhyOffsetInConsumeQueue() throws RocksDBException; ++ ++ /** ++ * get min logic offset of specific topic-queueId in consumeQueue ++ * @param topic ++ * @param queueId ++ * @return the min logic offset of specific topic-queueId in consumeQueue ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ long getMinOffsetInQueue(final String topic, final int queueId) throws RocksDBException; ++ ++ /** ++ * get max logic offset of specific topic-queueId in consumeQueue ++ * @param topic ++ * @param queueId ++ * @return the max logic offset of specific topic-queueId in consumeQueue ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ long getMaxOffsetInQueue(final String topic, final int queueId) throws RocksDBException; ++ ++ /** ++ * Get the message whose timestamp is the smallest, greater than or equal to the given time and when there are more ++ * than one message satisfy the condition, decide which one to return based on boundaryType. ++ * @param timestamp timestamp ++ * @param boundaryType Lower or Upper ++ * @return the offset(index) ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException; ++ ++ /** ++ * find or create the consumeQueue ++ * @param topic ++ * @param queueId ++ * @return the consumeQueue ++ */ ++ ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId); ++ ++ /** ++ * find the consumeQueueMap of topic ++ * @param topic ++ * @return the consumeQueueMap of topic ++ */ ++ ConcurrentMap findConsumeQueueMap(String topic); ++ ++ /** ++ * get the total size of all consumeQueue ++ * @return the total size of all consumeQueue ++ */ ++ long getTotalSize(); ++ ++ /** ++ * Get store time from commitlog by cqUnit ++ * @param cqUnit ++ * @return ++ */ ++ long getStoreTime(CqUnit cqUnit); ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java +new file mode 100644 +index 000000000..d6291d908 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java +@@ -0,0 +1,76 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.util.ArrayList; ++import java.util.List; ++import java.util.Map; ++ ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.MixAll; ++import org.apache.rocketmq.common.message.MessageConst; ++import org.apache.rocketmq.common.topic.TopicValidator; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.config.MessageStoreConfig; ++ ++public class MultiDispatch { ++ ++ public static String lmqQueueKey(String queueName) { ++ StringBuilder keyBuilder = new StringBuilder(); ++ keyBuilder.append(queueName); ++ keyBuilder.append('-'); ++ int queueId = 0; ++ keyBuilder.append(queueId); ++ return keyBuilder.toString(); ++ } ++ ++ public static boolean isNeedHandleMultiDispatch(MessageStoreConfig messageStoreConfig, String topic) { ++ return messageStoreConfig.isEnableMultiDispatch() ++ && !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) ++ && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX) ++ && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); ++ } ++ ++ public static boolean checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, DispatchRequest dispatchRequest) { ++ if (!isNeedHandleMultiDispatch(messageStoreConfig, dispatchRequest.getTopic())) { ++ return false; ++ } ++ Map prop = dispatchRequest.getPropertiesMap(); ++ if (prop == null || prop.isEmpty()) { ++ return false; ++ } ++ String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); ++ String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); ++ if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { ++ return false; ++ } ++ return true; ++ } ++ ++ public static List checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, List dispatchRequests) { ++ if (!messageStoreConfig.isEnableMultiDispatch() || dispatchRequests == null || dispatchRequests.size() == 0) { ++ return null; ++ } ++ List result = new ArrayList<>(); ++ for (DispatchRequest dispatchRequest : dispatchRequests) { ++ if (checkMultiDispatchQueue(messageStoreConfig, dispatchRequest)) { ++ result.add(dispatchRequest); ++ } ++ } ++ return dispatchRequests; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java +index 2545bbf52..8da374828 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java +@@ -41,6 +41,10 @@ public class QueueOffsetOperator { + return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); + } + ++ public Long getTopicQueueNextOffset(String topicQueueKey) { ++ return this.topicQueueTable.get(topicQueueKey); ++ } ++ + public void increaseQueueOffset(String topicQueueKey, short messageNum) { + Long queueOffset = ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); + topicQueueTable.put(topicQueueKey, queueOffset + messageNum); +@@ -63,6 +67,10 @@ public class QueueOffsetOperator { + return ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); + } + ++ public Long getLmqTopicQueueNextOffset(String topicQueueKey) { ++ return this.lmqTopicQueueTable.get(topicQueueKey); ++ } ++ + public void increaseLmqOffset(String topicQueueKey, short messageNum) { + Long lmqOffset = ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); + this.lmqTopicQueueTable.put(topicQueueKey, lmqOffset + messageNum); +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java +new file mode 100644 +index 000000000..759be395d +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java +@@ -0,0 +1,437 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.nio.ByteBuffer; ++import java.util.List; ++ ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.common.MixAll; ++import org.apache.rocketmq.common.Pair; ++import org.apache.rocketmq.common.attribute.CQType; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.message.MessageAccessor; ++import org.apache.rocketmq.common.message.MessageConst; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.ConsumeQueue; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.MessageFilter; ++import org.apache.rocketmq.store.MessageStore; ++import org.rocksdb.RocksDBException; ++ ++public class RocksDBConsumeQueue implements ConsumeQueueInterface { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); ++ ++ private final MessageStore messageStore; ++ private final String topic; ++ private final int queueId; ++ ++ public RocksDBConsumeQueue(final MessageStore messageStore, final String topic, final int queueId) { ++ this.messageStore = messageStore; ++ this.topic = topic; ++ this.queueId = queueId; ++ } ++ ++ public RocksDBConsumeQueue(final String topic, final int queueId) { ++ this.messageStore = null; ++ this.topic = topic; ++ this.queueId = queueId; ++ } ++ ++ @Override ++ public boolean load() { ++ return true; ++ } ++ ++ @Override ++ public void recover() { ++ // ignore ++ } ++ ++ @Override ++ public void checkSelf() { ++ // ignore ++ } ++ ++ @Override ++ public boolean flush(final int flushLeastPages) { ++ return true; ++ } ++ ++ @Override ++ public void destroy() { ++ // ignore ++ } ++ ++ @Override ++ public void truncateDirtyLogicFiles(long maxCommitLogPos) { ++ // ignored ++ } ++ ++ @Override ++ public int deleteExpiredFile(long minCommitLogPos) { ++ return 0; ++ } ++ ++ @Override ++ public long rollNextFile(long nextBeginOffset) { ++ return 0; ++ } ++ ++ @Override ++ public boolean isFirstFileAvailable() { ++ return true; ++ } ++ ++ @Override ++ public boolean isFirstFileExist() { ++ return true; ++ } ++ ++ @Override ++ public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) { ++ // ignore ++ } ++ ++ @Override ++ public void cleanSwappedMap(long forceCleanSwapIntervalMs) { ++ // ignore ++ } ++ ++ @Override ++ public long getMaxOffsetInQueue() { ++ try { ++ return this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getMaxOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e); ++ return 0; ++ } ++ } ++ ++ @Override ++ public long getMessageTotalInQueue() { ++ try { ++ long maxOffsetInQueue = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); ++ long minOffsetInQueue = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId); ++ return maxOffsetInQueue - minOffsetInQueue; ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getMessageTotalInQueue Failed. topic: {}, queueId: {}, {}", topic, queueId, e); ++ } ++ return -1; ++ } ++ ++ /** ++ * We already implement it in RocksDBConsumeQueueStore. ++ * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime ++ * @param timestamp timestamp ++ * @return ++ */ ++ @Override ++ public long getOffsetInQueueByTime(long timestamp) { ++ return 0; ++ } ++ ++ /** ++ * We already implement it in RocksDBConsumeQueueStore. ++ * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime ++ * @param timestamp timestamp ++ * @param boundaryType Lower or Upper ++ * @return ++ */ ++ @Override ++ public long getOffsetInQueueByTime(long timestamp, BoundaryType boundaryType) { ++ return 0; ++ } ++ ++ @Override ++ public long getMaxPhysicOffset() { ++ Long maxPhyOffset = this.messageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue(topic, queueId); ++ return maxPhyOffset == null ? -1 : maxPhyOffset; ++ } ++ ++ @Override ++ public long getMinLogicOffset() { ++ return 0; ++ } ++ ++ @Override ++ public CQType getCQType() { ++ return CQType.RocksDBCQ; ++ } ++ ++ @Override ++ public long getTotalSize() { ++ // ignored ++ return 0; ++ } ++ ++ @Override ++ public int getUnitSize() { ++ // attention: unitSize should equal to 'ConsumeQueue.CQ_STORE_UNIT_SIZE' ++ return ConsumeQueue.CQ_STORE_UNIT_SIZE; ++ } ++ ++ /** ++ * Ignored, we already implement this method ++ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) ++ */ ++ @Override ++ public void correctMinOffset(long minCommitLogOffset) { ++ ++ } ++ ++ /** ++ * Ignored, in rocksdb mode, we build cq in RocksDBConsumeQueueStore ++ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore#putMessagePosition() ++ */ ++ @Override ++ public void putMessagePositionInfoWrapper(DispatchRequest request) { ++ ++ } ++ ++ @Override ++ public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg) throws RocksDBException { ++ String topicQueueKey = getTopic() + "-" + getQueueId(); ++ Long queueOffset = queueOffsetOperator.getTopicQueueNextOffset(topicQueueKey); ++ if (queueOffset == null) { ++ // we will recover topic queue table from rocksdb when we use it. ++ queueOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); ++ queueOffsetOperator.updateQueueOffset(topicQueueKey, queueOffset); ++ } ++ msg.setQueueOffset(queueOffset); ++ ++ // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), ++ // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. ++ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { ++ return; ++ } ++ String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); ++ if (StringUtils.isBlank(multiDispatchQueue)) { ++ return; ++ } ++ String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); ++ Long[] queueOffsets = new Long[queues.length]; ++ for (int i = 0; i < queues.length; i++) { ++ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { ++ String key = MultiDispatch.lmqQueueKey(queues[i]); ++ queueOffsets[i] = queueOffsetOperator.getLmqTopicQueueNextOffset(key); ++ } ++ } ++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, ++ StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); ++ msg.removeWaitStorePropertyString(); ++ } ++ ++ @Override ++ public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg, short messageNum) { ++ String topicQueueKey = getTopic() + "-" + getQueueId(); ++ queueOffsetOperator.increaseQueueOffset(topicQueueKey, messageNum); ++ ++ // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), ++ // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. ++ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { ++ return; ++ } ++ String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); ++ if (StringUtils.isBlank(multiDispatchQueue)) { ++ return; ++ } ++ String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); ++ for (int i = 0; i < queues.length; i++) { ++ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { ++ String key = MultiDispatch.lmqQueueKey(queues[i]); ++ queueOffsetOperator.increaseLmqOffset(key, (short) 1); ++ } ++ } ++ } ++ ++ @Override ++ public long estimateMessageCount(long from, long to, MessageFilter filter) { ++ // todo ++ return 0; ++ } ++ ++ @Override ++ public long getMinOffsetInQueue() { ++ return this.messageStore.getMinOffsetInQueue(this.topic, this.queueId); ++ } ++ ++ private int pullNum(long cqOffset, long maxCqOffset) { ++ long diffLong = maxCqOffset - cqOffset; ++ if (diffLong < Integer.MAX_VALUE) { ++ int diffInt = (int) diffLong; ++ return diffInt > 16 ? 16 : diffInt; ++ } ++ return 16; ++ } ++ ++ @Override ++ public ReferredIterator iterateFrom(final long startIndex) { ++ try { ++ long maxCqOffset = getMaxOffsetInQueue(); ++ if (startIndex < maxCqOffset) { ++ int num = pullNum(startIndex, maxCqOffset); ++ return iterateFrom0(startIndex, num); ++ } ++ } catch (RocksDBException e) { ++ log.error("[RocksDBConsumeQueue] iterateFrom error!", e); ++ } ++ return null; ++ } ++ ++ @Override ++ public ReferredIterator iterateFrom(long startIndex, int count) throws RocksDBException { ++ long maxCqOffset = getMaxOffsetInQueue(); ++ if (startIndex < maxCqOffset) { ++ int num = Math.min((int)(maxCqOffset - startIndex), count); ++ return iterateFrom0(startIndex, num); ++ } ++ return null; ++ } ++ ++ @Override ++ public CqUnit get(long index) { ++ Pair pair = getCqUnitAndStoreTime(index); ++ return pair == null ? null : pair.getObject1(); ++ } ++ ++ @Override ++ public Pair getCqUnitAndStoreTime(long index) { ++ ByteBuffer byteBuffer; ++ try { ++ byteBuffer = this.messageStore.getQueueStore().get(topic, queueId, index); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e); ++ return null; ++ } ++ if (byteBuffer == null || byteBuffer.remaining() < RocksDBConsumeQueueTable.CQ_UNIT_SIZE) { ++ return null; ++ } ++ long phyOffset = byteBuffer.getLong(); ++ int size = byteBuffer.getInt(); ++ long tagCode = byteBuffer.getLong(); ++ long messageStoreTime = byteBuffer.getLong(); ++ return new Pair<>(new CqUnit(index, phyOffset, size, tagCode), messageStoreTime); ++ } ++ ++ @Override ++ public Pair getEarliestUnitAndStoreTime() { ++ try { ++ long minOffset = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId); ++ return getCqUnitAndStoreTime(minOffset); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getEarliestUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e); ++ } ++ return null; ++ } ++ ++ @Override ++ public CqUnit getEarliestUnit() { ++ Pair pair = getEarliestUnitAndStoreTime(); ++ return pair == null ? null : pair.getObject1(); ++ } ++ ++ @Override ++ public CqUnit getLatestUnit() { ++ try { ++ long maxOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); ++ return get(maxOffset); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getLatestUnit Failed. topic: {}, queueId: {}, {}", topic, queueId, e.getMessage()); ++ } ++ return null; ++ } ++ ++ @Override ++ public long getLastOffset() { ++ return getMaxPhysicOffset(); ++ } ++ ++ private ReferredIterator iterateFrom0(final long startIndex, final int count) throws RocksDBException { ++ List byteBufferList = this.messageStore.getQueueStore().rangeQuery(topic, queueId, startIndex, count); ++ if (byteBufferList == null || byteBufferList.isEmpty()) { ++ if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ log.warn("iterateFrom0 - find nothing, startIndex:{}, count:{}", startIndex, count); ++ } ++ return null; ++ } ++ return new RocksDBConsumeQueueIterator(byteBufferList, startIndex); ++ } ++ ++ @Override ++ public String getTopic() { ++ return topic; ++ } ++ ++ @Override ++ public int getQueueId() { ++ return queueId; ++ } ++ ++ private class RocksDBConsumeQueueIterator implements ReferredIterator { ++ private final List byteBufferList; ++ private final long startIndex; ++ private final int totalCount; ++ private int currentIndex; ++ ++ public RocksDBConsumeQueueIterator(final List byteBufferList, final long startIndex) { ++ this.byteBufferList = byteBufferList; ++ this.startIndex = startIndex; ++ this.totalCount = byteBufferList.size(); ++ this.currentIndex = 0; ++ } ++ ++ @Override ++ public boolean hasNext() { ++ return this.currentIndex < this.totalCount; ++ } ++ ++ @Override ++ public CqUnit next() { ++ if (!hasNext()) { ++ return null; ++ } ++ final int currentIndex = this.currentIndex; ++ final ByteBuffer byteBuffer = this.byteBufferList.get(currentIndex); ++ CqUnit cqUnit = new CqUnit(this.startIndex + currentIndex, byteBuffer.getLong(), byteBuffer.getInt(), byteBuffer.getLong()); ++ this.currentIndex++; ++ return cqUnit; ++ } ++ ++ @Override ++ public void remove() { ++ throw new UnsupportedOperationException("remove"); ++ } ++ ++ @Override ++ public void release() { ++ } ++ ++ @Override ++ public CqUnit nextAndRelease() { ++ try { ++ return next(); ++ } finally { ++ release(); ++ } ++ } ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java +new file mode 100644 +index 000000000..6fa66282e +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java +@@ -0,0 +1,641 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.nio.ByteBuffer; ++import java.util.HashMap; ++import java.util.HashSet; ++import java.util.Map; ++import java.util.Set; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++ ++import org.apache.rocketmq.common.Pair; ++import org.apache.rocketmq.common.TopicConfig; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.topic.TopicValidator; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.ConsumeQueue; ++import org.apache.rocketmq.store.DefaultMessageStore; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; ++import org.rocksdb.ColumnFamilyHandle; ++import org.rocksdb.RocksDBException; ++import org.rocksdb.RocksIterator; ++import org.rocksdb.WriteBatch; ++ ++import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; ++import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; ++ ++public class RocksDBConsumeQueueOffsetTable { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); ++ private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); ++ ++ private static final byte[] MAX_BYTES = "max".getBytes(CHARSET_UTF8); ++ private static final byte[] MIN_BYTES = "min".getBytes(CHARSET_UTF8); ++ ++ /** ++ * Rocksdb ConsumeQueue's Offset unit. Format: ++ * ++ * 

++     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬─────────────┐
++     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  Max(Min) │  CTRL_1   │   QueueId   │
++     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │  (4 Bytes)  │
++     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴─────────────┤
++     * │                                                    Key Unit                                                   │
++     * │                                                                                                               │
++     *
++ * ++ * 
++     * ┌─────────────────────────────┬────────────────────────┐
++     * │  CommitLog Physical Offset  │   ConsumeQueue Offset  │
++     * │        (8 Bytes)            │    (8 Bytes)           │
++     * ├─────────────────────────────┴────────────────────────┤
++     * │                     Value Unit                       │
++     * │                                                      │
++     *
++ * ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) = 16 Bytes ++ */ ++ private static final int OFFSET_PHY_OFFSET = 0; ++ private static final int OFFSET_CQ_OFFSET = 8; ++ /** ++ * ++ * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐ ++ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ Max(Min) │ CTRL_1 │ QueueId │ ++ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │ ++ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴─────────────┤ ++ */ ++ private static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4; ++ private static final int OFFSET_VALUE_LENGTH = 8 + 8; ++ ++ /** ++ * We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread. ++ * @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread. ++ * If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of ++ * RocksDBConsumeQueueOffsetTable to find it. ++ */ ++ private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC; ++ private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(CHARSET_UTF8); ++ private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length; ++ private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN); ++ private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN]; ++ private final ByteBuffer maxPhyOffsetBB; ++ static { ++ buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true); ++ INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); ++ INNER_CHECKPOINT_TOPIC.get(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY); ++ } ++ ++ private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; ++ private final ConsumeQueueRocksDBStorage rocksDBStorage; ++ private final DefaultMessageStore messageStore; ++ ++ private ColumnFamilyHandle offsetCFH; ++ ++ /** ++ * Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them ++ * from heap to avoid accessing rocksdb. ++ * @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset --> topicQueueMaxCqOffset ++ * @see ConsumeQueue#getMinLogicOffset(), minLogicOffset --> topicQueueMinOffset ++ */ ++ private final Map topicQueueMinOffset; ++ private final Map topicQueueMaxCqOffset; ++ ++ public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable, ++ ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { ++ this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable; ++ this.rocksDBStorage = rocksDBStorage; ++ this.messageStore = messageStore; ++ this.topicQueueMinOffset = new ConcurrentHashMap(1024); ++ this.topicQueueMaxCqOffset = new ConcurrentHashMap(1024); ++ ++ this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8); ++ } ++ ++ public void load() { ++ this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle(); ++ } ++ ++ public void updateTempTopicQueueMaxOffset(final Pair offsetBBPair, ++ final byte[] topicBytes, final DispatchRequest request, ++ final Map> tempTopicQueueMaxOffsetMap) { ++ buildOffsetKeyAndValueByteBuffer(offsetBBPair, topicBytes, request); ++ ByteBuffer topicQueueId = offsetBBPair.getObject1(); ++ ByteBuffer maxOffsetBB = offsetBBPair.getObject2(); ++ Pair old = tempTopicQueueMaxOffsetMap.get(topicQueueId); ++ if (old == null) { ++ tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair(maxOffsetBB, request)); ++ } else { ++ long oldMaxOffset = old.getObject1().getLong(OFFSET_CQ_OFFSET); ++ long maxOffset = maxOffsetBB.getLong(OFFSET_CQ_OFFSET); ++ if (maxOffset >= oldMaxOffset) { ++ ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset); ++ } ++ } ++ } ++ ++ public void putMaxPhyAndCqOffset(final Map> tempTopicQueueMaxOffsetMap, ++ final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { ++ for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { ++ writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1()); ++ } ++ ++ appendMaxPhyOffset(writeBatch, maxPhyOffset); ++ } ++ ++ public void putHeapMaxCqOffset(final Map> tempTopicQueueMaxOffsetMap) { ++ for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { ++ DispatchRequest request = entry.getValue().getObject2(); ++ putHeapMaxCqOffset(request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset()); ++ } ++ } ++ ++ /** ++ * When topic is deleted, we clean up its offset info in rocksdb. ++ * @param topic ++ * @param queueId ++ * @throws RocksDBException ++ */ ++ public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false); ++ byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array()); ++ Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; ++ ++ final ByteBuffer maxOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, true); ++ byte[] maxOffsetBytes = this.rocksDBStorage.getOffset(maxOffsetKey.array()); ++ Long endCQOffset = (maxOffsetBytes != null) ? ByteBuffer.wrap(maxOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; ++ ++ writeBatch.delete(this.offsetCFH, minOffsetKey.array()); ++ writeBatch.delete(this.offsetCFH, maxOffsetKey.array()); ++ ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ removeHeapMinCqOffset(topicQueueId); ++ removeHeapMaxCqOffset(topicQueueId); ++ ++ log.info("RocksDB offset table delete topic: {}, queueId: {}, minOffset: {}, maxOffset: {}", topic, queueId, ++ startCQOffset, endCQOffset); ++ } ++ ++ private void appendMaxPhyOffset(final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { ++ final ByteBuffer maxPhyOffsetBB = this.maxPhyOffsetBB; ++ maxPhyOffsetBB.position(0).limit(8); ++ maxPhyOffsetBB.putLong(maxPhyOffset); ++ maxPhyOffsetBB.flip(); ++ ++ INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); ++ writeBatch.put(this.offsetCFH, INNER_CHECKPOINT_TOPIC, maxPhyOffsetBB); ++ } ++ ++ public long getMaxPhyOffset() throws RocksDBException { ++ byte[] valueBytes = this.rocksDBStorage.getOffset(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY); ++ if (valueBytes == null) { ++ return 0; ++ } ++ ByteBuffer valueBB = ByteBuffer.wrap(valueBytes); ++ return valueBB.getLong(0); ++ } ++ ++ /** ++ * Traverse the offset table to find dirty topic ++ * @param existTopicSet ++ * @return ++ */ ++ public Map> iterateOffsetTable2FindDirty(final Set existTopicSet) { ++ Map> topicQueueIdToBeDeletedMap = new HashMap<>(); ++ ++ RocksIterator iterator = null; ++ try { ++ iterator = rocksDBStorage.seekOffsetCF(); ++ if (iterator == null) { ++ return topicQueueIdToBeDeletedMap; ++ } ++ for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { ++ byte[] key = iterator.key(); ++ byte[] value = iterator.value(); ++ if (key == null || key.length <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES ++ || value == null || value.length != OFFSET_VALUE_LENGTH) { ++ continue; ++ } ++ ByteBuffer keyBB = ByteBuffer.wrap(key); ++ int topicLen = keyBB.getInt(0); ++ byte[] topicBytes = new byte[topicLen]; ++ /** ++ * "Topic Bytes Array Size" + "CTRL_1" = 4 + 1 ++ */ ++ keyBB.position(4 + 1); ++ keyBB.get(topicBytes); ++ String topic = new String(topicBytes, CHARSET_UTF8); ++ if (TopicValidator.isSystemTopic(topic)) { ++ continue; ++ } ++ ++ /** ++ * "Topic Bytes Array Size" + "CTRL_1" + "Topic Bytes Array" + "CTRL_1" + "Max(min)" + "CTRL_1" ++ * = 4 + 1 + topicLen + 1 + 3 + 1 ++ */ ++ int queueId = keyBB.getInt(4 + 1 + topicLen + 1 + 3 + 1); ++ ++ if (!existTopicSet.contains(topic)) { ++ ByteBuffer valueBB = ByteBuffer.wrap(value); ++ long cqOffset = valueBB.getLong(OFFSET_CQ_OFFSET); ++ ++ Set topicQueueIdSet = topicQueueIdToBeDeletedMap.get(topic); ++ if (topicQueueIdSet == null) { ++ Set newSet = new HashSet<>(); ++ newSet.add(queueId); ++ topicQueueIdToBeDeletedMap.put(topic, newSet); ++ } else { ++ topicQueueIdSet.add(queueId); ++ } ++ ERROR_LOG.info("RocksDBConsumeQueueOffsetTable has dirty cqOffset. topic: {}, queueId: {}, cqOffset: {}", ++ topic, queueId, cqOffset); ++ } ++ } ++ } catch (Exception e) { ++ ERROR_LOG.error("iterateOffsetTable2MarkDirtyCQ Failed.", e); ++ } finally { ++ if (iterator != null) { ++ iterator.close(); ++ } ++ } ++ return topicQueueIdToBeDeletedMap; ++ } ++ ++ public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException { ++ Long maxCqOffset = getHeapMaxCqOffset(topic, queueId); ++ ++ if (maxCqOffset == null) { ++ final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); ++ maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null; ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, maxCqOffset != null ? maxCqOffset : -1L); ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, maxCqOffset); ++ } ++ } ++ ++ return maxCqOffset; ++ } ++ ++ /** ++ * truncate dirty offset in rocksdb ++ * @param offsetToTruncate ++ * @throws RocksDBException ++ */ ++ public void truncateDirty(long offsetToTruncate) throws RocksDBException { ++ correctMaxPyhOffset(offsetToTruncate); ++ ++ ConcurrentMap allTopicConfigMap = this.messageStore.getTopicConfigs(); ++ if (allTopicConfigMap == null) { ++ return; ++ } ++ for (TopicConfig topicConfig : allTopicConfigMap.values()) { ++ for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { ++ truncateDirtyOffset(topicConfig.getTopicName(), i); ++ } ++ } ++ } ++ ++ private Pair isMinOffsetOk(final String topic, final int queueId, final long minPhyOffset) throws RocksDBException { ++ PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); ++ if (phyAndCQOffset != null) { ++ final long phyOffset = phyAndCQOffset.getPhyOffset(); ++ final long cqOffset = phyAndCQOffset.getCqOffset(); ++ ++ return (phyOffset >= minPhyOffset) ? new Pair(true, cqOffset) : new Pair(false, cqOffset); ++ } ++ ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId); ++ if (byteBuffer == null) { ++ return new Pair(false, 0L); ++ } ++ final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); ++ final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); ++ if (phyOffset >= minPhyOffset) { ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ PhyAndCQOffset newPhyAndCQOffset = new PhyAndCQOffset(phyOffset, cqOffset); ++ this.topicQueueMinOffset.putIfAbsent(topicQueueId, newPhyAndCQOffset); ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("updateMinOffsetInQueue. {}, {}", topicQueueId, newPhyAndCQOffset); ++ } ++ return new Pair(true, cqOffset); ++ } ++ return new Pair(false, cqOffset); ++ } ++ ++ private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException { ++ final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); ++ if (byteBuffer == null) { ++ return; ++ } ++ ++ long maxPhyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); ++ long maxCqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); ++ long maxPhyOffsetInCQ = getMaxPhyOffset(); ++ ++ if (maxPhyOffset >= maxPhyOffsetInCQ) { ++ correctMaxCqOffset(topic, queueId, maxCqOffset, maxPhyOffsetInCQ); ++ Long newMaxCqOffset = getHeapMaxCqOffset(topic, queueId); ++ ROCKSDB_LOG.warn("truncateDirtyLogicFile topic: {}, queueId: {} from {} to {}", topic, queueId, ++ maxPhyOffset, newMaxCqOffset); ++ } ++ } ++ ++ private void correctMaxPyhOffset(long maxPhyOffset) throws RocksDBException { ++ if (!this.rocksDBStorage.hold()) { ++ return; ++ } ++ try { ++ WriteBatch writeBatch = new WriteBatch(); ++ long oldMaxPhyOffset = getMaxPhyOffset(); ++ if (oldMaxPhyOffset <= maxPhyOffset) { ++ return; ++ } ++ log.info("correctMaxPyhOffset, oldMaxPhyOffset={}, newMaxPhyOffset={}", oldMaxPhyOffset, maxPhyOffset); ++ appendMaxPhyOffset(writeBatch, maxPhyOffset); ++ this.rocksDBStorage.batchPut(writeBatch); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("correctMaxPyhOffset Failed.", e); ++ throw e; ++ } finally { ++ this.rocksDBStorage.release(); ++ } ++ } ++ ++ public long getMinCqOffset(String topic, int queueId) throws RocksDBException { ++ final long minPhyOffset = this.messageStore.getMinPhyOffset(); ++ Pair pair = isMinOffsetOk(topic, queueId, minPhyOffset); ++ final long cqOffset = pair.getObject2(); ++ if (!pair.getObject1() && correctMinCqOffset(topic, queueId, cqOffset, minPhyOffset)) { ++ PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); ++ if (phyAndCQOffset != null) { ++ if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("getMinOffsetInQueue miss heap. topic: {}, queueId: {}, old: {}, new: {}", ++ topic, queueId, cqOffset, phyAndCQOffset); ++ } ++ return phyAndCQOffset.getCqOffset(); ++ } ++ } ++ return cqOffset; ++ } ++ ++ public Long getMaxPhyOffset(String topic, int queueId) { ++ try { ++ ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); ++ if (byteBuffer != null) { ++ return byteBuffer.getLong(OFFSET_PHY_OFFSET); ++ } ++ } catch (Exception e) { ++ ERROR_LOG.info("getMaxPhyOffset error. topic: {}, queueId: {}", topic, queueId); ++ } ++ return null; ++ } ++ ++ private ByteBuffer getMinPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException { ++ return getPhyAndCqOffsetInKV(topic, queueId, false); ++ } ++ ++ private ByteBuffer getMaxPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException { ++ return getPhyAndCqOffsetInKV(topic, queueId, true); ++ } ++ ++ private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max); ++ ++ byte[] value = this.rocksDBStorage.getOffset(keyBB.array()); ++ return (value != null) ? ByteBuffer.wrap(value) : null; ++ } ++ ++ private String buildTopicQueueId(final String topic, final int queueId) { ++ return topic + "-" + queueId; ++ } ++ ++ private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, final long minCQOffset) { ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset); ++ this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset); ++ } ++ ++ private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxCQOffset) { ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ Long oldMaxCqOffset = this.topicQueueMaxCqOffset.put(topicQueueId, maxCQOffset); ++ if (oldMaxCqOffset != null && oldMaxCqOffset > maxCQOffset) { ++ ERROR_LOG.error("cqOffset invalid0. old: {}, now: {}", oldMaxCqOffset, maxCQOffset); ++ } ++ } ++ ++ private PhyAndCQOffset getHeapMinOffset(final String topic, final int queueId) { ++ return this.topicQueueMinOffset.get(buildTopicQueueId(topic, queueId)); ++ } ++ ++ private Long getHeapMaxCqOffset(final String topic, final int queueId) { ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ return this.topicQueueMaxCqOffset.get(topicQueueId); ++ } ++ ++ private PhyAndCQOffset removeHeapMinCqOffset(String topicQueueId) { ++ return this.topicQueueMinOffset.remove(topicQueueId); ++ } ++ ++ private Long removeHeapMaxCqOffset(String topicQueueId) { ++ return this.topicQueueMaxCqOffset.remove(topicQueueId); ++ } ++ ++ private void updateCqOffset(final String topic, final int queueId, final long phyOffset, ++ final long cqOffset, boolean max) throws RocksDBException { ++ if (!this.rocksDBStorage.hold()) { ++ return; ++ } ++ WriteBatch writeBatch = new WriteBatch(); ++ try { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max); ++ ++ final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset); ++ writeBatch.put(this.offsetCFH, offsetKey.array(), offsetValue.array()); ++ this.rocksDBStorage.batchPut(writeBatch); ++ ++ if (max) { ++ putHeapMaxCqOffset(topic, queueId, cqOffset); ++ } else { ++ putHeapMinCqOffset(topic, queueId, phyOffset, cqOffset); ++ } ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("updateCqOffset({}) failed.", max ? "max" : "min", e); ++ throw e; ++ } finally { ++ writeBatch.close(); ++ this.rocksDBStorage.release(); ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}", ++ max ? "max" : "min", topic, queueId, phyOffset, cqOffset); ++ } ++ } ++ } ++ ++ private boolean correctMaxCqOffset(final String topic, final int queueId, final long maxCQOffset, ++ final long maxPhyOffsetInCQ) throws RocksDBException { ++ // 'getMinOffsetInQueue' may correct minCqOffset and put it into heap ++ long minCQOffset = getMinCqOffset(topic, queueId); ++ PhyAndCQOffset minPhyAndCQOffset = getHeapMinOffset(topic, queueId); ++ if (minPhyAndCQOffset == null ++ || minPhyAndCQOffset.getCqOffset() != minCQOffset ++ || minPhyAndCQOffset.getPhyOffset() > maxPhyOffsetInCQ) { ++ ROCKSDB_LOG.info("[BUG] correctMaxCqOffset error! topic: {}, queueId: {}, maxPhyOffsetInCQ: {}, " ++ + "minCqOffset: {}, phyAndCQOffset: {}", ++ topic, queueId, maxPhyOffsetInCQ, minCQOffset, minPhyAndCQOffset); ++ throw new RocksDBException("correctMaxCqOffset error"); ++ } ++ ++ long high = maxCQOffset; ++ long low = minCQOffset; ++ PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, ++ low, maxPhyOffsetInCQ, false); ++ ++ long targetCQOffset = targetPhyAndCQOffset.getCqOffset(); ++ long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset(); ++ ++ if (targetCQOffset == -1) { ++ if (maxCQOffset != minCQOffset) { ++ updateCqOffset(topic, queueId, minPhyAndCQOffset.getPhyOffset(), minCQOffset, true); ++ } ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyAndCQOffset.getPhyOffset()); ++ } ++ return false; ++ } else { ++ updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, true); ++ return true; ++ } ++ } ++ ++ private boolean correctMinCqOffset(final String topic, final int queueId, ++ final long minCQOffset, final long minPhyOffset) throws RocksDBException { ++ final ByteBuffer maxBB = getMaxPhyAndCqOffsetInKV(topic, queueId); ++ if (maxBB == null) { ++ updateCqOffset(topic, queueId, minPhyOffset, 0L, false); ++ return true; ++ } ++ final long maxPhyOffset = maxBB.getLong(OFFSET_PHY_OFFSET); ++ final long maxCQOffset = maxBB.getLong(OFFSET_CQ_OFFSET); ++ ++ if (maxPhyOffset < minPhyOffset) { ++ updateCqOffset(topic, queueId, minPhyOffset, maxCQOffset + 1, false); ++ return true; ++ } ++ ++ long high = maxCQOffset; ++ long low = minCQOffset; ++ PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, low, ++ minPhyOffset, true); ++ long targetCQOffset = phyAndCQOffset.getCqOffset(); ++ long targetPhyOffset = phyAndCQOffset.getPhyOffset(); ++ ++ if (targetCQOffset == -1) { ++ if (maxCQOffset != minCQOffset) { ++ updateCqOffset(topic, queueId, maxPhyOffset, maxCQOffset, false); ++ } ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyOffset); ++ } ++ return false; ++ } else { ++ updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, false); ++ return true; ++ } ++ } ++ ++ public static Pair getOffsetByteBufferPair() { ++ ByteBuffer offsetKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN); ++ ByteBuffer offsetValue = ByteBuffer.allocateDirect(OFFSET_VALUE_LENGTH); ++ return new Pair<>(offsetKey, offsetValue); ++ } ++ ++ private void buildOffsetKeyAndValueByteBuffer(final Pair offsetBBPair, ++ final byte[] topicBytes, final DispatchRequest request) { ++ final ByteBuffer offsetKey = offsetBBPair.getObject1(); ++ buildOffsetKeyByteBuffer(offsetKey, topicBytes, request.getQueueId(), true); ++ ++ final ByteBuffer offsetValue = offsetBBPair.getObject2(); ++ buildOffsetValueByteBuffer(offsetValue, request.getCommitLogOffset(), request.getConsumeQueueOffset()); ++ } ++ ++ private ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) { ++ ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); ++ return byteBuffer; ++ } ++ ++ private void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) { ++ byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); ++ } ++ ++ private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, ++ final boolean max) { ++ byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1); ++ if (max) { ++ byteBuffer.put(MAX_BYTES); ++ } else { ++ byteBuffer.put(MIN_BYTES); ++ } ++ byteBuffer.put(CTRL_1).putInt(queueId); ++ byteBuffer.flip(); ++ } ++ ++ private void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { ++ byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH); ++ buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); ++ } ++ ++ private ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) { ++ final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH); ++ buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); ++ return byteBuffer; ++ } ++ ++ private void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { ++ byteBuffer.putLong(phyOffset).putLong(cqOffset); ++ byteBuffer.flip(); ++ } ++ ++ static class PhyAndCQOffset { ++ private final long phyOffset; ++ private final long cqOffset; ++ ++ public PhyAndCQOffset(final long phyOffset, final long cqOffset) { ++ this.phyOffset = phyOffset; ++ this.cqOffset = cqOffset; ++ } ++ ++ public long getPhyOffset() { ++ return this.phyOffset; ++ } ++ ++ public long getCqOffset() { ++ return this.cqOffset; ++ } ++ ++ @Override ++ public String toString() { ++ return "[cqOffset=" + cqOffset + ", phyOffset=" + phyOffset + "]"; ++ } ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +new file mode 100644 +index 000000000..78456cfcd +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +@@ -0,0 +1,441 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.io.File; ++import java.nio.ByteBuffer; ++import java.util.ArrayList; ++import java.util.HashMap; ++import java.util.List; ++import java.util.Map; ++import java.util.Set; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++import java.util.concurrent.Executors; ++import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.TimeUnit; ++ ++import org.apache.commons.io.FileUtils; ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.common.Pair; ++import org.apache.rocketmq.common.ThreadFactoryImpl; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.common.utils.DataConverter; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.DefaultMessageStore; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.config.BrokerRole; ++import org.apache.rocketmq.store.config.StorePathConfigHelper; ++import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; ++import org.rocksdb.RocksDBException; ++import org.rocksdb.WriteBatch; ++ ++public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { ++ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); ++ private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); ++ ++ public static final byte CTRL_0 = '\u0000'; ++ public static final byte CTRL_1 = '\u0001'; ++ public static final byte CTRL_2 = '\u0002'; ++ ++ private static final int BATCH_SIZE = 16; ++ public static final int MAX_KEY_LEN = 300; ++ ++ private final ScheduledExecutorService scheduledExecutorService; ++ private final String storePath; ++ ++ /** ++ * we use two tables with different ColumnFamilyHandle, called RocksDBConsumeQueueTable and RocksDBConsumeQueueOffsetTable. ++ * 1.RocksDBConsumeQueueTable uses to store CqUnit[physicalOffset, msgSize, tagHashCode, msgStoreTime] ++ * 2.RocksDBConsumeQueueOffsetTable uses to store physicalOffset and consumeQueueOffset(@see PhyAndCQOffset) of topic-queueId ++ */ ++ private final ConsumeQueueRocksDBStorage rocksDBStorage; ++ private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; ++ private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable; ++ ++ private final WriteBatch writeBatch; ++ private final List bufferDRList; ++ private final List> cqBBPairList; ++ private final List> offsetBBPairList; ++ private final Map> tempTopicQueueMaxOffsetMap; ++ private volatile boolean isCQError = false; ++ ++ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { ++ super(messageStore); ++ ++ this.storePath = StorePathConfigHelper.getStorePathConsumeQueue(messageStoreConfig.getStorePathRootDir()); ++ this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath, 4); ++ this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore); ++ this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); ++ ++ this.writeBatch = new WriteBatch(); ++ this.bufferDRList = new ArrayList(BATCH_SIZE); ++ this.cqBBPairList = new ArrayList(BATCH_SIZE); ++ this.offsetBBPairList = new ArrayList(BATCH_SIZE); ++ for (int i = 0; i < BATCH_SIZE; i++) { ++ this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); ++ this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); ++ } ++ ++ this.tempTopicQueueMaxOffsetMap = new HashMap<>(); ++ this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("RocksDBConsumeQueueStoreScheduledThread", messageStore.getBrokerIdentity())); ++ } ++ ++ @Override ++ public void start() { ++ log.info("RocksDB ConsumeQueueStore start!"); ++ this.scheduledExecutorService.scheduleAtFixedRate(() -> { ++ this.rocksDBStorage.statRocksdb(ROCKSDB_LOG); ++ }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS); ++ ++ this.scheduledExecutorService.scheduleWithFixedDelay(() -> { ++ cleanDirty(messageStore.getTopicConfigs().keySet()); ++ }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES); ++ } ++ ++ private void cleanDirty(final Set existTopicSet) { ++ try { ++ Map> topicQueueIdToBeDeletedMap = ++ this.rocksDBConsumeQueueOffsetTable.iterateOffsetTable2FindDirty(existTopicSet); ++ ++ for (Map.Entry> entry : topicQueueIdToBeDeletedMap.entrySet()) { ++ String topic = entry.getKey(); ++ for (int queueId : entry.getValue()) { ++ destroy(new RocksDBConsumeQueue(topic, queueId)); ++ } ++ } ++ } catch (Exception e) { ++ log.error("cleanUnusedTopic Failed.", e); ++ } ++ } ++ ++ @Override ++ public boolean load() { ++ boolean result = this.rocksDBStorage.start(); ++ this.rocksDBConsumeQueueTable.load(); ++ this.rocksDBConsumeQueueOffsetTable.load(); ++ log.info("load rocksdb consume queue {}.", result ? "OK" : "Failed"); ++ return result; ++ } ++ ++ @Override ++ public boolean loadAfterDestroy() { ++ return this.load(); ++ } ++ ++ @Override ++ public void recover() { ++ // ignored ++ } ++ ++ @Override ++ public boolean recoverConcurrently() { ++ return true; ++ } ++ ++ @Override ++ public boolean shutdown() { ++ this.scheduledExecutorService.shutdown(); ++ return shutdownInner(); ++ } ++ ++ private boolean shutdownInner() { ++ return this.rocksDBStorage.shutdown(); ++ } ++ ++ @Override ++ public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException { ++ if (request == null || this.bufferDRList.size() >= BATCH_SIZE) { ++ putMessagePosition(); ++ } ++ if (request != null) { ++ this.bufferDRList.add(request); ++ } ++ } ++ ++ public void putMessagePosition() throws RocksDBException { ++ final int maxRetries = 30; ++ for (int i = 0; i < maxRetries; i++) { ++ if (putMessagePosition0()) { ++ if (this.isCQError) { ++ this.messageStore.getRunningFlags().clearLogicsQueueError(); ++ this.isCQError = false; ++ } ++ return; ++ } else { ++ ERROR_LOG.warn("{} put cq Failed. retryTime: {}", i); ++ try { ++ Thread.sleep(100); ++ } catch (InterruptedException ignored) { ++ } ++ } ++ } ++ if (!this.isCQError) { ++ ERROR_LOG.error("[BUG] put CQ Failed."); ++ this.messageStore.getRunningFlags().makeLogicsQueueError(); ++ this.isCQError = true; ++ } ++ throw new RocksDBException("put CQ Failed"); ++ } ++ ++ private boolean putMessagePosition0() { ++ if (!this.rocksDBStorage.hold()) { ++ return false; ++ } ++ ++ final Map> tempTopicQueueMaxOffsetMap = this.tempTopicQueueMaxOffsetMap; ++ try { ++ final List bufferDRList = this.bufferDRList; ++ final int size = bufferDRList.size(); ++ if (size == 0) { ++ return true; ++ } ++ final List> cqBBPairList = this.cqBBPairList; ++ final List> offsetBBPairList = this.offsetBBPairList; ++ final WriteBatch writeBatch = this.writeBatch; ++ ++ long maxPhyOffset = 0; ++ for (int i = size - 1; i >= 0; i--) { ++ final DispatchRequest request = bufferDRList.get(i); ++ final byte[] topicBytes = request.getTopic().getBytes(DataConverter.CHARSET_UTF8); ++ ++ this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(cqBBPairList.get(i), topicBytes, request, writeBatch); ++ this.rocksDBConsumeQueueOffsetTable.updateTempTopicQueueMaxOffset(offsetBBPairList.get(i), ++ topicBytes, request, tempTopicQueueMaxOffsetMap); ++ ++ final int msgSize = request.getMsgSize(); ++ final long phyOffset = request.getCommitLogOffset(); ++ if (phyOffset + msgSize >= maxPhyOffset) { ++ maxPhyOffset = phyOffset + msgSize; ++ } ++ } ++ ++ this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset); ++ ++ // clear writeBatch in batchPut ++ this.rocksDBStorage.batchPut(writeBatch); ++ ++ this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap); ++ ++ long storeTimeStamp = bufferDRList.get(size - 1).getStoreTimestamp(); ++ if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE ++ || this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) { ++ this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp); ++ } ++ this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp); ++ ++ notifyMessageArriveAndClear(); ++ return true; ++ } catch (Exception e) { ++ ERROR_LOG.error("putMessagePosition0 Failed.", e); ++ return false; ++ } finally { ++ tempTopicQueueMaxOffsetMap.clear(); ++ this.rocksDBStorage.release(); ++ } ++ } ++ ++ private void notifyMessageArriveAndClear() { ++ final List bufferDRList = this.bufferDRList; ++ try { ++ for (DispatchRequest dp : bufferDRList) { ++ this.messageStore.notifyMessageArriveIfNecessary(dp); ++ } ++ } catch (Exception e) { ++ ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e); ++ } finally { ++ bufferDRList.clear(); ++ } ++ } ++ ++ @Override ++ public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { ++ return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num); ++ } ++ ++ @Override ++ public ByteBuffer get(final String topic, final int queueId, final long cqOffset) throws RocksDBException { ++ return this.rocksDBConsumeQueueTable.getCQInKV(topic, queueId, cqOffset); ++ } ++ ++ /** ++ * Ignored, we do not need to recover topicQueueTable and correct minLogicOffset. Because we will correct them ++ * when we use them, we call it lazy correction. ++ * @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short) ++ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) ++ */ ++ @Override ++ public void recoverOffsetTable(long minPhyOffset) { ++ ++ } ++ ++ @Override ++ public void destroy() { ++ try { ++ shutdownInner(); ++ FileUtils.deleteDirectory(new File(this.storePath)); ++ } catch (Exception e) { ++ ERROR_LOG.error("destroy cq Failed. {}", this.storePath, e); ++ } ++ } ++ ++ @Override ++ public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException { ++ String topic = consumeQueue.getTopic(); ++ int queueId = consumeQueue.getQueueId(); ++ if (StringUtils.isEmpty(topic) || queueId < 0 || !this.rocksDBStorage.hold()) { ++ return; ++ } ++ ++ WriteBatch writeBatch = new WriteBatch(); ++ try { ++ this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch); ++ this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch); ++ ++ this.rocksDBStorage.batchPut(writeBatch); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("kv deleteTopic {} Failed.", topic, e); ++ throw e; ++ } finally { ++ writeBatch.close(); ++ this.rocksDBStorage.release(); ++ } ++ } ++ ++ @Override ++ public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { ++ try { ++ this.rocksDBStorage.flushWAL(); ++ } catch (Exception e) { ++ } ++ return true; ++ } ++ ++ @Override ++ public void checkSelf() { ++ // ignored ++ } ++ ++ @Override ++ public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) { ++ // ignored ++ return 0; ++ } ++ ++ /** ++ * We do not need to truncate dirty CQ in RocksDBConsumeQueueTable, Because dirty CQ in RocksDBConsumeQueueTable ++ * will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted. ++ * But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in ++ * RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset). ++ * @param offsetToTruncate ++ * @throws RocksDBException ++ */ ++ @Override ++ public void truncateDirty(long offsetToTruncate) throws RocksDBException { ++ long maxPhyOffsetInRocksdb = getMaxPhyOffsetInConsumeQueue(); ++ if (offsetToTruncate >= maxPhyOffsetInRocksdb) { ++ return; ++ } ++ ++ this.rocksDBConsumeQueueOffsetTable.truncateDirty(offsetToTruncate); ++ } ++ ++ @Override ++ public void cleanExpired(final long minPhyOffset) { ++ this.rocksDBStorage.manualCompaction(minPhyOffset); ++ } ++ ++ @Override ++ public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException { ++ final long minPhysicOffset = this.messageStore.getMinPhyOffset(); ++ long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); ++ Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); ++ if (high == null || high == -1) { ++ return 0; ++ } ++ return this.rocksDBConsumeQueueTable.binarySearchInCQByTime(topic, queueId, high, low, timestamp, minPhysicOffset); ++ } ++ ++ @Override ++ public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException { ++ Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); ++ return (maxOffset != null) ? maxOffset + 1 : 0; ++ } ++ ++ @Override ++ public long getMinOffsetInQueue(String topic, int queueId) throws RocksDBException { ++ return this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); ++ } ++ ++ @Override ++ public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) { ++ return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(topic, queueId); ++ } ++ ++ @Override ++ public long getMaxPhyOffsetInConsumeQueue() throws RocksDBException { ++ return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(); ++ } ++ ++ @Override ++ public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) { ++ ConcurrentMap map = this.consumeQueueTable.get(topic); ++ if (null == map) { ++ ConcurrentMap newMap = new ConcurrentHashMap<>(128); ++ ConcurrentMap oldMap = this.consumeQueueTable.putIfAbsent(topic, newMap); ++ if (oldMap != null) { ++ map = oldMap; ++ } else { ++ map = newMap; ++ } ++ } ++ ++ ConsumeQueueInterface logic = map.get(queueId); ++ if (logic != null) { ++ return logic; ++ } ++ ++ ConsumeQueueInterface newLogic = new RocksDBConsumeQueue(this.messageStore, topic, queueId); ++ ConsumeQueueInterface oldLogic = map.putIfAbsent(queueId, newLogic); ++ ++ return oldLogic != null ? oldLogic : newLogic; ++ } ++ ++ @Override ++ public long rollNextFile(ConsumeQueueInterface consumeQueue, long offset) { ++ return 0; ++ } ++ ++ @Override ++ public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) { ++ return true; ++ } ++ ++ @Override ++ public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) { ++ return true; ++ } ++ ++ @Override ++ public long getTotalSize() { ++ return 0; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java +new file mode 100644 +index 000000000..0a735ea27 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java +@@ -0,0 +1,312 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.nio.ByteBuffer; ++import java.util.ArrayList; ++import java.util.List; ++ ++import org.apache.rocketmq.common.Pair; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.DefaultMessageStore; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset; ++import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; ++import org.rocksdb.ColumnFamilyHandle; ++import org.rocksdb.RocksDBException; ++import org.rocksdb.WriteBatch; ++ ++import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; ++import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_0; ++import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; ++import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_2; ++ ++/** ++ * We use RocksDBConsumeQueueTable to store cqUnit. ++ */ ++public class RocksDBConsumeQueueTable { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); ++ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); ++ ++ /** ++ * Rocksdb ConsumeQueue's store unit. Format: ++ * ++ * 
++     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬───────────────────────┐
++     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  QueueId  │  CTRL_1   │  ConsumeQueue Offset  │
++     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │     (8 Bytes)         │
++     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴───────────────────────┤
++     * │                                                    Key Unit                                                             │
++     * │                                                                                                                         │
++     *
++ * ++ * 
++     * ┌─────────────────────────────┬───────────────────┬──────────────────┬──────────────────┐
++     * │  CommitLog Physical Offset  │      Body Size    │   Tag HashCode   │  Msg Store Time  │
++     * │        (8 Bytes)            │      (4 Bytes)    │    (8 Bytes)     │    (8 Bytes)     │
++     * ├─────────────────────────────┴───────────────────┴──────────────────┴──────────────────┤
++     * │                                                    Value Unit                         │
++     * │                                                                                       │
++     *
++ * ConsumeQueue's store unit. Size: ++ * CommitLog Physical Offset(8) + Body Size(4) + Tag HashCode(8) + Msg Store Time(8) = 28 Bytes ++ */ ++ private static final int PHY_OFFSET_OFFSET = 0; ++ private static final int PHY_MSG_LEN_OFFSET = 8; ++ private static final int MSG_TAG_HASHCODE_OFFSET = 12; ++ private static final int MSG_STORE_TIME_SIZE_OFFSET = 20; ++ public static final int CQ_UNIT_SIZE = 8 + 4 + 8 + 8; ++ ++ /** ++ * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬───────────────────────┐ ++ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_1 │ ConsumeQueue Offset │ ++ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ (8 Bytes) │ ++ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴───────────────────────┤ ++ */ ++ private static final int CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1 + 8; ++ ++ /** ++ * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────────────┐ ++ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_0(CTRL_2) │ ++ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ ++ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────────────┤ ++ */ ++ private static final int DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1; ++ ++ private final ConsumeQueueRocksDBStorage rocksDBStorage; ++ private final DefaultMessageStore messageStore; ++ ++ private ColumnFamilyHandle defaultCFH; ++ ++ public RocksDBConsumeQueueTable(ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { ++ this.rocksDBStorage = rocksDBStorage; ++ this.messageStore = messageStore; ++ } ++ ++ public void load() { ++ this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle(); ++ } ++ ++ public void buildAndPutCQByteBuffer(final Pair cqBBPair, ++ final byte[] topicBytes, final DispatchRequest request, final WriteBatch writeBatch) throws RocksDBException { ++ final ByteBuffer cqKey = cqBBPair.getObject1(); ++ buildCQKeyByteBuffer(cqKey, topicBytes, request.getQueueId(), request.getConsumeQueueOffset()); ++ ++ final ByteBuffer cqValue = cqBBPair.getObject2(); ++ buildCQValueByteBuffer(cqValue, request.getCommitLogOffset(), request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp()); ++ ++ writeBatch.put(this.defaultCFH, cqKey, cqValue); ++ } ++ ++ public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset); ++ byte[] value = this.rocksDBStorage.getCQ(keyBB.array()); ++ return (value != null) ? ByteBuffer.wrap(value) : null; ++ } ++ ++ public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final List defaultCFHList = new ArrayList(num); ++ final ByteBuffer[] resultList = new ByteBuffer[num]; ++ final List kvIndexList = new ArrayList(num); ++ final List kvKeyList = new ArrayList(num); ++ for (int i = 0; i < num; i++) { ++ final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i); ++ kvIndexList.add(i); ++ kvKeyList.add(keyBB.array()); ++ defaultCFHList.add(this.defaultCFH); ++ } ++ int keyNum = kvIndexList.size(); ++ if (keyNum > 0) { ++ List kvValueList = this.rocksDBStorage.multiGet(defaultCFHList, kvKeyList); ++ final int valueNum = kvValueList.size(); ++ if (keyNum != valueNum) { ++ throw new RocksDBException("rocksdb bug, multiGet"); ++ } ++ for (int i = 0; i < valueNum; i++) { ++ byte[] value = kvValueList.get(i); ++ if (value == null) { ++ continue; ++ } ++ ByteBuffer byteBuffer = ByteBuffer.wrap(value); ++ resultList[kvIndexList.get(i)] = byteBuffer; ++ } ++ } ++ ++ final int resultSize = resultList.length; ++ List bbValueList = new ArrayList(resultSize); ++ for (int i = 0; i < resultSize; i++) { ++ ByteBuffer byteBuffer = resultList[i]; ++ if (byteBuffer == null) { ++ break; ++ } ++ bbValueList.add(byteBuffer); ++ } ++ return bbValueList; ++ } ++ ++ /** ++ * When topic is deleted, we clean up its CqUnit in rocksdb. ++ * @param topic ++ * @param queueId ++ * @throws RocksDBException ++ */ ++ public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId); ++ final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId); ++ ++ writeBatch.deleteRange(this.defaultCFH, cqStartKey.array(), cqEndKey.array()); ++ ++ log.info("Rocksdb consumeQueue table delete topic. {}, {}", topic, queueId); ++ } ++ ++ public long binarySearchInCQByTime(String topic, int queueId, long high, long low, long timestamp, ++ long minPhysicOffset) throws RocksDBException { ++ long result = 0; ++ long targetOffset = -1L, leftOffset = -1L, rightOffset = -1L; ++ long leftValue = -1L, rightValue = -1L; ++ while (high >= low) { ++ long midOffset = low + ((high - low) >>> 1); ++ ByteBuffer byteBuffer = getCQInKV(topic, queueId, midOffset); ++ if (byteBuffer == null) { ++ ERROR_LOG.warn("binarySearchInCQByTimeStamp Failed. topic: {}, queueId: {}, timestamp: {}, result: null", ++ topic, queueId, timestamp); ++ low = midOffset + 1; ++ continue; ++ } ++ ++ long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET); ++ if (phyOffset < minPhysicOffset) { ++ low = midOffset + 1; ++ leftOffset = midOffset; ++ continue; ++ } ++ long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET); ++ if (storeTime < 0) { ++ return 0; ++ } else if (storeTime == timestamp) { ++ targetOffset = midOffset; ++ break; ++ } else if (storeTime > timestamp) { ++ high = midOffset - 1; ++ rightOffset = midOffset; ++ rightValue = storeTime; ++ } else { ++ low = midOffset + 1; ++ leftOffset = midOffset; ++ leftValue = storeTime; ++ } ++ } ++ if (targetOffset != -1) { ++ result = targetOffset; ++ } else { ++ if (leftValue == -1) { ++ result = rightOffset; ++ } else if (rightValue == -1) { ++ result = leftOffset; ++ } else { ++ result = Math.abs(timestamp - leftValue) > Math.abs(timestamp - rightValue) ? rightOffset : leftOffset; ++ } ++ } ++ return result; ++ } ++ ++ public PhyAndCQOffset binarySearchInCQ(String topic, int queueId, long high, long low, long targetPhyOffset, ++ boolean min) throws RocksDBException { ++ long resultCQOffset = -1L; ++ long resultPhyOffset = -1L; ++ while (high >= low) { ++ long midCQOffset = low + ((high - low) >>> 1); ++ ByteBuffer byteBuffer = getCQInKV(topic, queueId, midCQOffset); ++ if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("binarySearchInCQ. {}, {}, {}, {}, {}", topic, queueId, midCQOffset, low, high); ++ } ++ if (byteBuffer == null) { ++ low = midCQOffset + 1; ++ continue; ++ } ++ ++ final long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET); ++ if (phyOffset == targetPhyOffset) { ++ if (min) { ++ resultCQOffset = midCQOffset; ++ resultPhyOffset = phyOffset; ++ } ++ break; ++ } else if (phyOffset > targetPhyOffset) { ++ high = midCQOffset - 1; ++ if (min) { ++ resultCQOffset = midCQOffset; ++ resultPhyOffset = phyOffset; ++ } ++ } else { ++ low = midCQOffset + 1; ++ if (!min) { ++ resultCQOffset = midCQOffset; ++ resultPhyOffset = phyOffset; ++ } ++ } ++ } ++ return new PhyAndCQOffset(resultPhyOffset, resultCQOffset); ++ } ++ ++ public static Pair getCQByteBufferPair() { ++ ByteBuffer cqKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN); ++ ByteBuffer cqValue = ByteBuffer.allocateDirect(CQ_UNIT_SIZE); ++ return new Pair<>(cqKey, cqValue); ++ } ++ ++ private ByteBuffer buildCQKeyByteBuffer(final byte[] topicBytes, final int queueId, final long cqOffset) { ++ final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset); ++ return byteBuffer; ++ } ++ ++ private void buildCQKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) { ++ byteBuffer.position(0).limit(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset); ++ } ++ ++ private void buildCQKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) { ++ byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(CTRL_1).putLong(cqOffset); ++ byteBuffer.flip(); ++ } ++ ++ private void buildCQValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, final long tagsCode, final long storeTimestamp) { ++ byteBuffer.position(0).limit(CQ_UNIT_SIZE); ++ buildCQValueByteBuffer0(byteBuffer, phyOffset, msgSize, tagsCode, storeTimestamp); ++ } ++ ++ private void buildCQValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, ++ final long tagsCode, final long storeTimestamp) { ++ byteBuffer.putLong(phyOffset).putInt(msgSize).putLong(tagsCode).putLong(storeTimestamp); ++ byteBuffer.flip(); ++ } ++ ++ private ByteBuffer buildDeleteCQKey(final boolean start, final byte[] topicBytes, final int queueId) { ++ final ByteBuffer byteBuffer = ByteBuffer.allocate(DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ ++ byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(start ? CTRL_0 : CTRL_2); ++ byteBuffer.flip(); ++ return byteBuffer; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java +new file mode 100644 +index 000000000..aa796c4d3 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java +@@ -0,0 +1,47 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.rocksdb; ++ ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.MessageStore; ++import org.rocksdb.AbstractCompactionFilter; ++import org.rocksdb.AbstractCompactionFilterFactory; ++import org.rocksdb.RemoveConsumeQueueCompactionFilter; ++ ++public class ConsumeQueueCompactionFilterFactory extends AbstractCompactionFilterFactory { ++ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); ++ private final MessageStore messageStore; ++ ++ public ConsumeQueueCompactionFilterFactory(final MessageStore messageStore) { ++ this.messageStore = messageStore; ++ } ++ ++ @Override ++ public String name() { ++ return "ConsumeQueueCompactionFilterFactory"; ++ } ++ ++ @Override ++ public RemoveConsumeQueueCompactionFilter createCompactionFilter(final AbstractCompactionFilter.Context context) { ++ long minPhyOffset = this.messageStore.getMinPhyOffset(); ++ LOGGER.info("manualCompaction minPhyOffset: {}, isFull: {}, isManual: {}", ++ minPhyOffset, context.isFullCompaction(), context.isManualCompaction()); ++ return new RemoveConsumeQueueCompactionFilter(minPhyOffset); ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java +new file mode 100644 +index 000000000..362684560 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java +@@ -0,0 +1,133 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.rocksdb; ++ ++import java.util.ArrayList; ++import java.util.List; ++ ++import org.apache.rocketmq.common.UtilAll; ++import org.apache.rocketmq.common.config.AbstractRocksDBStorage; ++import org.apache.rocketmq.common.utils.DataConverter; ++import org.apache.rocketmq.store.MessageStore; ++import org.rocksdb.ColumnFamilyDescriptor; ++import org.rocksdb.ColumnFamilyHandle; ++import org.rocksdb.ColumnFamilyOptions; ++import org.rocksdb.CompactRangeOptions; ++import org.rocksdb.ReadOptions; ++import org.rocksdb.RocksDB; ++import org.rocksdb.RocksDBException; ++import org.rocksdb.RocksIterator; ++import org.rocksdb.WriteBatch; ++import org.rocksdb.WriteOptions; ++ ++public class ConsumeQueueRocksDBStorage extends AbstractRocksDBStorage { ++ private final MessageStore messageStore; ++ private volatile ColumnFamilyHandle offsetCFHandle; ++ ++ public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath, final int prefixLen) { ++ this.messageStore = messageStore; ++ this.dbPath = dbPath; ++ this.readOnly = false; ++ } ++ ++ private void initOptions() { ++ this.options = RocksDBOptionsFactory.createDBOptions(); ++ ++ this.writeOptions = new WriteOptions(); ++ this.writeOptions.setSync(false); ++ this.writeOptions.setDisableWAL(true); ++ this.writeOptions.setNoSlowdown(true); ++ ++ this.totalOrderReadOptions = new ReadOptions(); ++ this.totalOrderReadOptions.setPrefixSameAsStart(false); ++ this.totalOrderReadOptions.setTotalOrderSeek(false); ++ ++ this.compactRangeOptions = new CompactRangeOptions(); ++ this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce); ++ this.compactRangeOptions.setAllowWriteStall(true); ++ this.compactRangeOptions.setExclusiveManualCompaction(false); ++ this.compactRangeOptions.setChangeLevel(true); ++ this.compactRangeOptions.setTargetLevel(-1); ++ this.compactRangeOptions.setMaxSubcompactions(4); ++ } ++ ++ @Override ++ protected boolean postLoad() { ++ try { ++ UtilAll.ensureDirOK(this.dbPath); ++ ++ initOptions(); ++ ++ final List cfDescriptors = new ArrayList(); ++ ++ ColumnFamilyOptions cqCfOptions = RocksDBOptionsFactory.createCQCFOptions(this.messageStore); ++ this.cfOptions.add(cqCfOptions); ++ cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cqCfOptions)); ++ ++ ColumnFamilyOptions offsetCfOptions = RocksDBOptionsFactory.createOffsetCFOptions(); ++ this.cfOptions.add(offsetCfOptions); ++ cfDescriptors.add(new ColumnFamilyDescriptor("offset".getBytes(DataConverter.CHARSET_UTF8), offsetCfOptions)); ++ ++ final List cfHandles = new ArrayList(); ++ open(cfDescriptors, cfHandles); ++ ++ this.defaultCFHandle = cfHandles.get(0); ++ this.offsetCFHandle = cfHandles.get(1); ++ } catch (final Exception e) { ++ LOGGER.error("postLoad Failed. {}", this.dbPath, e); ++ return false; ++ } ++ return true; ++ } ++ ++ @Override ++ protected void preShutdown() { ++ this.offsetCFHandle.close(); ++ } ++ ++ public byte[] getCQ(final byte[] keyBytes) throws RocksDBException { ++ return get(this.defaultCFHandle, this.totalOrderReadOptions, keyBytes); ++ } ++ ++ public byte[] getOffset(final byte[] keyBytes) throws RocksDBException { ++ return get(this.offsetCFHandle, this.totalOrderReadOptions, keyBytes); ++ } ++ ++ public List multiGet(final List cfhList, final List keys) throws RocksDBException { ++ return multiGet(this.totalOrderReadOptions, cfhList, keys); ++ } ++ ++ public void batchPut(final WriteBatch batch) throws RocksDBException { ++ batchPut(this.writeOptions, batch); ++ } ++ ++ public void manualCompaction(final long minPhyOffset) { ++ try { ++ manualCompaction(minPhyOffset, this.compactRangeOptions); ++ } catch (Exception e) { ++ LOGGER.error("manualCompaction Failed. minPhyOffset: {}", minPhyOffset, e); ++ } ++ } ++ ++ public RocksIterator seekOffsetCF() { ++ return this.db.newIterator(this.offsetCFHandle, this.totalOrderReadOptions); ++ } ++ ++ public ColumnFamilyHandle getOffsetCFHandle() { ++ return this.offsetCFHandle; ++ } ++} +\ No newline at end of file +diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +new file mode 100644 +index 000000000..a3a99d334 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +@@ -0,0 +1,161 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.rocksdb; ++ ++import org.apache.rocketmq.common.config.ConfigRocksDBStorage; ++import org.apache.rocketmq.store.MessageStore; ++import org.rocksdb.BlockBasedTableConfig; ++import org.rocksdb.BloomFilter; ++import org.rocksdb.ColumnFamilyOptions; ++import org.rocksdb.CompactionOptionsUniversal; ++import org.rocksdb.CompactionStopStyle; ++import org.rocksdb.CompactionStyle; ++import org.rocksdb.CompressionType; ++import org.rocksdb.DBOptions; ++import org.rocksdb.DataBlockIndexType; ++import org.rocksdb.IndexType; ++import org.rocksdb.InfoLogLevel; ++import org.rocksdb.LRUCache; ++import org.rocksdb.RateLimiter; ++import org.rocksdb.SkipListMemTableConfig; ++import org.rocksdb.Statistics; ++import org.rocksdb.StatsLevel; ++import org.rocksdb.StringAppendOperator; ++import org.rocksdb.WALRecoveryMode; ++import org.rocksdb.util.SizeUnit; ++ ++public class RocksDBOptionsFactory { ++ ++ public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageStore) { ++ BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). ++ setFormatVersion(5). ++ setIndexType(IndexType.kBinarySearch). ++ setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash). ++ setDataBlockHashTableUtilRatio(0.75). ++ setBlockSize(32 * SizeUnit.KB). ++ setMetadataBlockSize(4 * SizeUnit.KB). ++ setFilterPolicy(new BloomFilter(16, false)). ++ setCacheIndexAndFilterBlocks(false). ++ setCacheIndexAndFilterBlocksWithHighPriority(true). ++ setPinL0FilterAndIndexBlocksInCache(false). ++ setPinTopLevelIndexAndFilter(true). ++ setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false)). ++ setWholeKeyFiltering(true); ++ ++ ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); ++ CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal(); ++ compactionOption.setSizeRatio(100). ++ setMaxSizeAmplificationPercent(25). ++ setAllowTrivialMove(true). ++ setMinMergeWidth(2). ++ setMaxMergeWidth(Integer.MAX_VALUE). ++ setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize). ++ setCompressionSizePercent(-1); ++ return columnFamilyOptions.setMaxWriteBufferNumber(4). ++ setWriteBufferSize(128 * SizeUnit.MB). ++ setMinWriteBufferNumberToMerge(1). ++ setTableFormatConfig(blockBasedTableConfig). ++ setMemTableConfig(new SkipListMemTableConfig()). ++ setCompressionType(CompressionType.LZ4_COMPRESSION). ++ setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION). ++ setNumLevels(7). ++ setCompactionStyle(CompactionStyle.UNIVERSAL). ++ setCompactionOptionsUniversal(compactionOption). ++ setMaxCompactionBytes(100 * SizeUnit.GB). ++ setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB). ++ setHardPendingCompactionBytesLimit(256 * SizeUnit.GB). ++ setLevel0FileNumCompactionTrigger(2). ++ setLevel0SlowdownWritesTrigger(8). ++ setLevel0StopWritesTrigger(10). ++ setTargetFileSizeBase(256 * SizeUnit.MB). ++ setTargetFileSizeMultiplier(2). ++ setMergeOperator(new StringAppendOperator()). ++ setCompactionFilterFactory(new ConsumeQueueCompactionFilterFactory(messageStore)). ++ setReportBgIoStats(true). ++ setOptimizeFiltersForHits(true); ++ } ++ ++ public static ColumnFamilyOptions createOffsetCFOptions() { ++ BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). ++ setFormatVersion(5). ++ setIndexType(IndexType.kBinarySearch). ++ setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch). ++ setBlockSize(32 * SizeUnit.KB). ++ setFilterPolicy(new BloomFilter(16, false)). ++ setCacheIndexAndFilterBlocks(false). ++ setCacheIndexAndFilterBlocksWithHighPriority(true). ++ setPinL0FilterAndIndexBlocksInCache(false). ++ setPinTopLevelIndexAndFilter(true). ++ setBlockCache(new LRUCache(128 * SizeUnit.MB, 8, false)). ++ setWholeKeyFiltering(true); ++ ++ ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); ++ return columnFamilyOptions.setMaxWriteBufferNumber(4). ++ setWriteBufferSize(64 * SizeUnit.MB). ++ setMinWriteBufferNumberToMerge(1). ++ setTableFormatConfig(blockBasedTableConfig). ++ setMemTableConfig(new SkipListMemTableConfig()). ++ setCompressionType(CompressionType.NO_COMPRESSION). ++ setNumLevels(7). ++ setCompactionStyle(CompactionStyle.LEVEL). ++ setLevel0FileNumCompactionTrigger(2). ++ setLevel0SlowdownWritesTrigger(8). ++ setLevel0StopWritesTrigger(10). ++ setTargetFileSizeBase(64 * SizeUnit.MB). ++ setTargetFileSizeMultiplier(2). ++ setMaxBytesForLevelBase(256 * SizeUnit.MB). ++ setMaxBytesForLevelMultiplier(2). ++ setMergeOperator(new StringAppendOperator()). ++ setInplaceUpdateSupport(true); ++ } ++ ++ /** ++ * Create a rocksdb db options, the user must take care to close it after closing db. ++ * @return ++ */ ++ public static DBOptions createDBOptions() { ++ //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide ++ // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java ++ DBOptions options = new DBOptions(); ++ Statistics statistics = new Statistics(); ++ statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS); ++ return options. ++ setDbLogDir(ConfigRocksDBStorage.getDBLogDir()). ++ setInfoLogLevel(InfoLogLevel.INFO_LEVEL). ++ setWalRecoveryMode(WALRecoveryMode.PointInTimeRecovery). ++ setManualWalFlush(true). ++ setMaxTotalWalSize(0). ++ setWalSizeLimitMB(0). ++ setWalTtlSeconds(0). ++ setCreateIfMissing(true). ++ setCreateMissingColumnFamilies(true). ++ setMaxOpenFiles(-1). ++ setMaxLogFileSize(1 * SizeUnit.GB). ++ setKeepLogFileNum(5). ++ setMaxManifestFileSize(1 * SizeUnit.GB). ++ setAllowConcurrentMemtableWrite(false). ++ setStatistics(statistics). ++ setAtomicFlush(true). ++ setMaxBackgroundJobs(32). ++ setMaxSubcompactions(8). ++ setParanoidChecks(true). ++ setDelayedWriteRate(16 * SizeUnit.MB). ++ setRateLimiter(new RateLimiter(100 * SizeUnit.MB)). ++ setUseDirectIoForFlushAndCompaction(false). ++ setUseDirectReads(false); ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +index ac4c61cd6..3ab51a26d 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +@@ -57,7 +57,6 @@ import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +-import org.apache.rocketmq.store.ConsumeQueue; + import org.apache.rocketmq.store.DefaultMessageStore; + import org.apache.rocketmq.store.MessageStore; + import org.apache.rocketmq.store.PutMessageResult; +@@ -66,6 +65,9 @@ import org.apache.rocketmq.store.config.BrokerRole; + import org.apache.rocketmq.store.config.MessageStoreConfig; + import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; ++import org.apache.rocketmq.store.queue.ConsumeQueueInterface; ++import org.apache.rocketmq.store.queue.CqUnit; ++import org.apache.rocketmq.store.queue.ReferredIterator; + import org.apache.rocketmq.store.stats.BrokerStatsManager; + import org.apache.rocketmq.store.util.PerfCounter; + +@@ -333,7 +335,7 @@ public class TimerMessageStore { + // if not, use cq offset. + long msgQueueOffset = messageExt.getQueueOffset(); + int queueId = messageExt.getQueueId(); +- ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); ++ ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); + if (null == cq) { + return msgQueueOffset; + } +@@ -346,15 +348,18 @@ public class TimerMessageStore { + offsetPy, sizePy); + break; + } +- SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(tmpOffset); +- if (null == bufferCQ) { +- // offset in msg may be greater than offset of cq. +- tmpOffset -= 1; +- continue; +- } ++ ReferredIterator iterator = null; + try { +- long offsetPyTemp = bufferCQ.getByteBuffer().getLong(); +- int sizePyTemp = bufferCQ.getByteBuffer().getInt(); ++ iterator = cq.iterateFrom(tmpOffset); ++ CqUnit cqUnit = null; ++ if (null == iterator || (cqUnit = iterator.next()) == null) { ++ // offset in msg may be greater than offset of cq. ++ tmpOffset -= 1; ++ continue; ++ } ++ ++ long offsetPyTemp = cqUnit.getPos(); ++ int sizePyTemp = cqUnit.getSize(); + if (offsetPyTemp == offsetPy && sizePyTemp == sizePy) { + LOGGER.info("reviseQueueOffset check cq offset ok. {}, {}, {}", + tmpOffset, offsetPyTemp, sizePyTemp); +@@ -365,7 +370,9 @@ public class TimerMessageStore { + } catch (Throwable e) { + LOGGER.error("reviseQueueOffset check cq offset error.", e); + } finally { +- bufferCQ.release(); ++ if (iterator != null) { ++ iterator.release(); ++ } + } + } + +@@ -633,7 +640,7 @@ public class TimerMessageStore { + if (!isRunningEnqueue()) { + return false; + } +- ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); ++ ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); + if (null == cq) { + return false; + } +@@ -643,18 +650,22 @@ public class TimerMessageStore { + currQueueOffset = cq.getMinOffsetInQueue(); + } + long offset = currQueueOffset; +- SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(offset); +- if (null == bufferCQ) { +- return false; +- } ++ ReferredIterator iterator = null; + try { ++ iterator = cq.iterateFrom(offset); ++ if (null == iterator) { ++ return false; ++ } ++ + int i = 0; +- for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) { ++ while (iterator.hasNext()) { ++ i++; + perfCounterTicks.startTick("enqueue_get"); + try { +- long offsetPy = bufferCQ.getByteBuffer().getLong(); +- int sizePy = bufferCQ.getByteBuffer().getInt(); +- bufferCQ.getByteBuffer().getLong(); //tags code ++ CqUnit cqUnit = iterator.next(); ++ long offsetPy = cqUnit.getPos(); ++ int sizePy = cqUnit.getSize(); ++ cqUnit.getTagsCode(); //tags code + MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy); + if (null == msgExt) { + perfCounterTicks.getCounter("enqueue_get_miss"); +@@ -663,7 +674,7 @@ public class TimerMessageStore { + lastEnqueueButExpiredStoreTime = msgExt.getStoreTimestamp(); + long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); + // use CQ offset, not offset in Message +- msgExt.setQueueOffset(offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE)); ++ msgExt.setQueueOffset(offset + i); + TimerRequest timerRequest = new TimerRequest(offsetPy, sizePy, delayedTime, System.currentTimeMillis(), MAGIC_DEFAULT, msgExt); + // System.out.printf("build enqueue request, %s%n", timerRequest); + while (!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) { +@@ -687,14 +698,16 @@ public class TimerMessageStore { + if (!isRunningEnqueue()) { + return false; + } +- currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); ++ currQueueOffset = offset + i; + } +- currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); ++ currQueueOffset = offset + i; + return i > 0; + } catch (Exception e) { + LOGGER.error("Unknown exception in enqueuing", e); + } finally { +- bufferCQ.release(); ++ if (iterator != null) { ++ iterator.release(); ++ } + } + return false; + } +@@ -1642,7 +1655,7 @@ public class TimerMessageStore { + if (System.currentTimeMillis() - start > storeConfig.getTimerProgressLogIntervalMs()) { + start = System.currentTimeMillis(); + long tmpQueueOffset = currQueueOffset; +- ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0); ++ ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); + long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); + TimerMessageStore.LOGGER.info("[{}]Timer progress-check commitRead:[{}] currRead:[{}] currWrite:[{}] readBehind:{} currReadOffset:{} offsetBehind:{} behindMaster:{} " + + "enqPutQueue:{} deqGetQueue:{} deqPutQueue:{} allCongestNum:{} enqExpiredStoreTime:{}", +@@ -1685,7 +1698,7 @@ public class TimerMessageStore { + + public long getEnqueueBehindMessages() { + long tmpQueueOffset = currQueueOffset; +- ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0); ++ ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); + long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); + return maxOffsetInQueue - tmpQueueOffset; + } +diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +index 12d1e5723..1d09ca86e 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java ++++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +@@ -428,9 +428,10 @@ public class DefaultMessageStoreTest { + + private long getStoreTime(CqUnit cqUnit) { + try { +- Method getStoreTime = getDefaultMessageStore().getClass().getDeclaredMethod("getStoreTime", CqUnit.class); ++ Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass(); ++ Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class); + getStoreTime.setAccessible(true); +- return (long) getStoreTime.invoke(getDefaultMessageStore(), cqUnit); ++ return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } +diff --git a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java +index 85626a332..2447bbf68 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java ++++ b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java +@@ -28,9 +28,11 @@ import org.apache.rocketmq.common.message.MessageConst; + import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExtBrokerInner; + import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.apache.rocketmq.store.queue.MultiDispatch; + import org.junit.After; + import org.junit.Before; + import org.junit.Test; ++import org.rocksdb.RocksDBException; + + import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue; + import static org.junit.Assert.assertEquals; +@@ -69,15 +71,15 @@ public class MultiDispatchTest { + } + + @Test +- public void queueKey() { ++ public void lmqQueueKey() { + MessageExtBrokerInner messageExtBrokerInner = mock(MessageExtBrokerInner.class); + when(messageExtBrokerInner.getQueueId()).thenReturn(2); +- String ret = consumeQueue.queueKey("%LMQ%lmq123", messageExtBrokerInner); ++ String ret = MultiDispatch.lmqQueueKey("%LMQ%lmq123"); + assertEquals(ret, "%LMQ%lmq123-0"); + } + + @Test +- public void wrapMultiDispatch() { ++ public void wrapMultiDispatch() throws RocksDBException { + MessageExtBrokerInner messageExtBrokerInner = buildMessageMultiQueue(); + messageStore.assignOffset(messageExtBrokerInner); + assertEquals(messageExtBrokerInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), "0,0"); +diff --git a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java +new file mode 100644 +index 000000000..acf5edf51 +--- /dev/null ++++ b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java +@@ -0,0 +1,1060 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.store; ++ ++import java.io.File; ++import java.io.RandomAccessFile; ++import java.lang.reflect.InvocationTargetException; ++import java.lang.reflect.Method; ++import java.net.InetAddress; ++import java.net.InetSocketAddress; ++import java.net.SocketAddress; ++import java.net.UnknownHostException; ++import java.nio.MappedByteBuffer; ++import java.nio.channels.FileChannel; ++import java.nio.channels.OverlappingFileLockException; ++import java.nio.charset.StandardCharsets; ++import java.util.ArrayList; ++import java.util.Arrays; ++import java.util.HashSet; ++import java.util.List; ++import java.util.Map; ++import java.util.Random; ++import java.util.UUID; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicInteger; ++ ++import com.google.common.collect.Sets; ++ ++import org.apache.rocketmq.common.BrokerConfig; ++import org.apache.rocketmq.common.MixAll; ++import org.apache.rocketmq.common.TopicConfig; ++import org.apache.rocketmq.common.UtilAll; ++import org.apache.rocketmq.common.message.MessageBatch; ++import org.apache.rocketmq.common.message.MessageDecoder; ++import org.apache.rocketmq.common.message.MessageExt; ++import org.apache.rocketmq.common.message.MessageExtBatch; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.store.config.BrokerRole; ++import org.apache.rocketmq.store.config.FlushDiskType; ++import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.apache.rocketmq.store.config.StorePathConfigHelper; ++import org.apache.rocketmq.store.queue.ConsumeQueueInterface; ++import org.apache.rocketmq.store.queue.CqUnit; ++import org.apache.rocketmq.store.stats.BrokerStatsManager; ++import org.assertj.core.util.Strings; ++import org.junit.After; ++import org.junit.Assert; ++import org.junit.Before; ++import org.junit.Test; ++import org.junit.runner.RunWith; ++import org.mockito.junit.MockitoJUnitRunner; ++ ++import static org.assertj.core.api.Assertions.assertThat; ++import static org.junit.Assert.assertEquals; ++import static org.junit.Assert.assertTrue; ++ ++@RunWith(MockitoJUnitRunner.class) ++public class RocksDBMessageStoreTest { ++ private final String storeMessage = "Once, there was a chance for me!"; ++ private final String messageTopic = "FooBar"; ++ private final String storeType = StoreType.DEFAULT_ROCKSDB.getStoreType(); ++ private int queueTotal = 100; ++ private AtomicInteger queueId = new AtomicInteger(0); ++ private SocketAddress bornHost; ++ private SocketAddress storeHost; ++ private byte[] messageBody; ++ private MessageStore messageStore; ++ ++ @Before ++ public void init() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); ++ bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); ++ ++ messageStore = buildMessageStore(); ++ boolean load = messageStore.load(); ++ assertTrue(load); ++ messageStore.start(); ++ } ++ ++ @Test(expected = OverlappingFileLockException.class) ++ public void test_repeat_restart() throws Exception { ++ if (notExecuted()) { ++ throw new OverlappingFileLockException(); ++ } ++ queueTotal = 1; ++ messageBody = storeMessage.getBytes(); ++ ++ MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); ++ messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8); ++ messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4); ++ messageStoreConfig.setMaxHashSlotNum(100); ++ messageStoreConfig.setMaxIndexNum(100 * 10); ++ messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir") + File.separator + "store"); ++ messageStoreConfig.setHaListenPort(0); ++ MessageStore master = new RocksDBMessageStore(messageStoreConfig, null, new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>()); ++ ++ boolean load = master.load(); ++ assertTrue(load); ++ ++ try { ++ master.start(); ++ master.start(); ++ } finally { ++ master.shutdown(); ++ master.destroy(); ++ } ++ } ++ ++ @After ++ public void destroy() { ++ if (notExecuted()) { ++ return; ++ } ++ messageStore.shutdown(); ++ messageStore.destroy(); ++ ++ MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); ++ File file = new File(messageStoreConfig.getStorePathRootDir()); ++ UtilAll.deleteFile(file); ++ } ++ ++ private MessageStore buildMessageStore() throws Exception { ++ return buildMessageStore(null, ""); ++ } ++ ++ private MessageStore buildMessageStore(String storePathRootDir, String topic) throws Exception { ++ MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); ++ messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10); ++ messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10); ++ messageStoreConfig.setMaxHashSlotNum(10000); ++ messageStoreConfig.setMaxIndexNum(100 * 100); ++ messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH); ++ messageStoreConfig.setFlushIntervalConsumeQueue(1); ++ messageStoreConfig.setStoreType(storeType); ++ messageStoreConfig.setHaListenPort(0); ++ if (Strings.isNullOrEmpty(storePathRootDir)) { ++ UUID uuid = UUID.randomUUID(); ++ storePathRootDir = System.getProperty("java.io.tmpdir") + File.separator + "store-" + uuid.toString(); ++ } ++ messageStoreConfig.setStorePathRootDir(storePathRootDir); ++ ConcurrentMap topicConfigTable = new ConcurrentHashMap<>(); ++ topicConfigTable.put(topic, new TopicConfig(topic, 1, 1)); ++ return new RocksDBMessageStore(messageStoreConfig, ++ new BrokerStatsManager("simpleTest", true), ++ new MyMessageArrivingListener(), ++ new BrokerConfig(), topicConfigTable); ++ } ++ ++ @Test ++ public void testWriteAndRead() { ++ if (notExecuted()) { ++ return; ++ } ++ long ipv4HostMsgs = 10; ++ long ipv6HostMsgs = 10; ++ long totalMsgs = ipv4HostMsgs + ipv6HostMsgs; ++ queueTotal = 1; ++ messageBody = storeMessage.getBytes(); ++ for (long i = 0; i < ipv4HostMsgs; i++) { ++ messageStore.putMessage(buildMessage()); ++ } ++ ++ for (long i = 0; i < ipv6HostMsgs; i++) { ++ messageStore.putMessage(buildIPv6HostMessage()); ++ } ++ ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ for (long i = 0; i < totalMsgs; i++) { ++ GetMessageResult result = messageStore.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); ++ assertThat(result).isNotNull(); ++ result.release(); ++ } ++ verifyThatMasterIsFunctional(totalMsgs, messageStore); ++ } ++ ++ @Test ++ public void testLookMessageByOffset_OffsetIsFirst() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = new Random().nextInt(10); ++ String topic = "FooBar"; ++ int firstOffset = 0; ++ AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); ++ AppendMessageResult firstResult = appendMessageResultArray[0]; ++ ++ MessageExt messageExt = messageStore.lookMessageByOffset(firstResult.getWroteOffset()); ++ MessageExt messageExt1 = getDefaultMessageStore().lookMessageByOffset(firstResult.getWroteOffset(), firstResult.getWroteBytes()); ++ ++ assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset)); ++ assertThat(new String(messageExt1.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset)); ++ } ++ ++ @Test ++ public void testLookMessageByOffset_OffsetIsLast() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = new Random().nextInt(10); ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); ++ int lastIndex = totalCount - 1; ++ AppendMessageResult lastResult = appendMessageResultArray[lastIndex]; ++ ++ MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastResult.getWroteOffset(), lastResult.getWroteBytes()); ++ ++ assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, lastIndex)); ++ } ++ ++ @Test ++ public void testLookMessageByOffset_OffsetIsOutOfBound() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = new Random().nextInt(10); ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); ++ long lastOffset = getMaxOffset(appendMessageResultArray); ++ ++ MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastOffset); ++ ++ assertThat(messageExt).isNull(); ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); ++ for (AppendMessageResult appendMessageResult : appendMessageResults) { ++ long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp()); ++ CqUnit cqUnit = consumeQueue.get(offset); ++ assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset()); ++ assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes()); ++ } ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_TimestampIsSkewing() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ int skewing = 2; ++ ++ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); ++ for (AppendMessageResult appendMessageResult : appendMessageResults) { ++ long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing); ++ CqUnit cqUnit = consumeQueue.get(offset); ++ assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset()); ++ assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes()); ++ } ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_TimestampSkewingIsLarge() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ int skewing = 20000; ++ ++ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); ++ for (AppendMessageResult appendMessageResult : appendMessageResults) { ++ long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing); ++ CqUnit cqUnit = consumeQueue.get(offset); ++ assertThat(cqUnit.getPos()).isEqualTo(appendMessageResults[0].getWroteOffset()); ++ assertThat(cqUnit.getSize()).isEqualTo(appendMessageResults[0].getWroteBytes()); ++ } ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_ConsumeQueueNotFound1() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ int wrongQueueId = 1; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); ++ //Thread.sleep(10); ++ ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ long offset = messageStore.getOffsetInQueueByTime(topic, wrongQueueId, appendMessageResults[0].getStoreTimestamp()); ++ ++ assertThat(offset).isEqualTo(0); ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_ConsumeQueueNotFound2() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ int wrongQueueId = 1; ++ String topic = "FooBar"; ++ putMessages(totalCount, topic, queueId, false); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, 0); ++ ++ assertThat(messageStoreTimeStamp).isEqualTo(-1); ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_ConsumeQueueOffsetNotExist() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ int wrongQueueId = 1; ++ String topic = "FooBar"; ++ putMessages(totalCount, topic, queueId, true); ++ //Thread.sleep(10); ++ ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, -1); ++ ++ assertThat(messageStoreTimeStamp).isEqualTo(-1); ++ } ++ ++ @Test ++ public void testGetMessageStoreTimeStamp() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); ++ int minOffsetInQueue = (int) consumeQueue.getMinOffsetInQueue(); ++ for (int i = minOffsetInQueue; i < consumeQueue.getMaxOffsetInQueue(); i++) { ++ long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, queueId, i); ++ assertThat(messageStoreTimeStamp).isEqualTo(appendMessageResults[i].getStoreTimestamp()); ++ } ++ } ++ ++ @Test ++ public void testGetStoreTime_ParamIsNull() { ++ if (notExecuted()) { ++ return; ++ } ++ long storeTime = getStoreTime(null); ++ ++ assertThat(storeTime).isEqualTo(-1); ++ } ++ ++ @Test ++ public void testGetStoreTime_EverythingIsOk() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, queueId); ++ ++ for (int i = 0; i < totalCount; i++) { ++ CqUnit cqUnit = consumeQueue.get(i); ++ long storeTime = getStoreTime(cqUnit); ++ assertThat(storeTime).isEqualTo(appendMessageResults[i].getStoreTimestamp()); ++ } ++ } ++ ++ @Test ++ public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() { ++ if (notExecuted()) { ++ return; ++ } ++ long phyOffset = -10; ++ int size = 138; ++ CqUnit cqUnit = new CqUnit(0, phyOffset, size, 0); ++ long storeTime = getStoreTime(cqUnit); ++ ++ assertThat(storeTime).isEqualTo(-1); ++ } ++ ++ @Test ++ public void testPutMessage_whenMessagePropertyIsTooLong() { ++ if (notExecuted()) { ++ return; ++ } ++ String topicName = "messagePropertyIsTooLongTest"; ++ MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1); ++ assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); ++ assertEquals(0L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue()); ++ MessageExtBrokerInner normalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, 100); ++ assertEquals(messageStore.putMessage(normalMessage).getPutMessageStatus(), PutMessageStatus.PUT_OK); ++ assertEquals(1L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue()); ++ } ++ ++ private RocksDBMessageStore getDefaultMessageStore() { ++ return (RocksDBMessageStore) this.messageStore; ++ } ++ ++ private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId) { ++ return putMessages(totalCount, topic, queueId, false); ++ } ++ ++ private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId, boolean interval) { ++ AppendMessageResult[] appendMessageResultArray = new AppendMessageResult[totalCount]; ++ for (int i = 0; i < totalCount; i++) { ++ String messageBody = buildMessageBodyByOffset(storeMessage, i); ++ ++ MessageExtBrokerInner msgInner = ++ i < totalCount / 2 ? buildMessage(messageBody.getBytes(), topic) : buildIPv6HostMessage(messageBody.getBytes(), topic); ++ msgInner.setQueueId(queueId); ++ PutMessageResult result = messageStore.putMessage(msgInner); ++ appendMessageResultArray[i] = result.getAppendMessageResult(); ++ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); ++ if (interval) { ++ try { ++ Thread.sleep(10); ++ } catch (InterruptedException e) { ++ throw new RuntimeException("Thread sleep ERROR"); ++ } ++ } ++ } ++ return appendMessageResultArray; ++ } ++ ++ private long getMaxOffset(AppendMessageResult[] appendMessageResultArray) { ++ if (appendMessageResultArray == null) { ++ return 0; ++ } ++ AppendMessageResult last = appendMessageResultArray[appendMessageResultArray.length - 1]; ++ return last.getWroteOffset() + last.getWroteBytes(); ++ } ++ ++ private String buildMessageBodyByOffset(String message, long i) { ++ return String.format("%s offset %d", message, i); ++ } ++ ++ private long getStoreTime(CqUnit cqUnit) { ++ try { ++ Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass(); ++ Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class); ++ getStoreTime.setAccessible(true); ++ return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit); ++ } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { ++ throw new RuntimeException(e); ++ } ++ } ++ ++ private MessageExtBrokerInner buildMessage(byte[] messageBody, String topic) { ++ MessageExtBrokerInner msg = new MessageExtBrokerInner(); ++ msg.setTopic(topic); ++ msg.setTags("TAG1"); ++ msg.setKeys("Hello"); ++ msg.setBody(messageBody); ++ msg.setKeys(String.valueOf(System.currentTimeMillis())); ++ msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); ++ msg.setSysFlag(0); ++ msg.setBornTimestamp(System.currentTimeMillis()); ++ msg.setStoreHost(storeHost); ++ msg.setBornHost(bornHost); ++ msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); ++ return msg; ++ } ++ ++ private MessageExtBrokerInner buildSpecifyLengthPropertyMessage(byte[] messageBody, String topic, int length) { ++ StringBuilder stringBuilder = new StringBuilder(); ++ Random random = new Random(); ++ for (int i = 0; i < length; i++) { ++ stringBuilder.append(random.nextInt(10)); ++ } ++ MessageExtBrokerInner msg = new MessageExtBrokerInner(); ++ msg.putUserProperty("test", stringBuilder.toString()); ++ msg.setTopic(topic); ++ msg.setTags("TAG1"); ++ msg.setKeys("Hello"); ++ msg.setBody(messageBody); ++ msg.setQueueId(0); ++ msg.setBornTimestamp(System.currentTimeMillis()); ++ msg.setStoreHost(storeHost); ++ msg.setBornHost(bornHost); ++ msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); ++ return msg; ++ } ++ ++ private MessageExtBrokerInner buildIPv6HostMessage(byte[] messageBody, String topic) { ++ MessageExtBrokerInner msg = new MessageExtBrokerInner(); ++ msg.setTopic(topic); ++ msg.setTags("TAG1"); ++ msg.setKeys("Hello"); ++ msg.setBody(messageBody); ++ msg.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); ++ msg.setKeys(String.valueOf(System.currentTimeMillis())); ++ msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); ++ msg.setSysFlag(0); ++ msg.setBornHostV6Flag(); ++ msg.setStoreHostAddressV6Flag(); ++ msg.setBornTimestamp(System.currentTimeMillis()); ++ try { ++ msg.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 0)); ++ } catch (UnknownHostException e) { ++ e.printStackTrace(); ++ assertThat(Boolean.FALSE).isTrue(); ++ } ++ ++ try { ++ msg.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 0)); ++ } catch (UnknownHostException e) { ++ e.printStackTrace(); ++ assertThat(Boolean.FALSE).isTrue(); ++ } ++ return msg; ++ } ++ ++ private MessageExtBrokerInner buildMessage() { ++ return buildMessage(messageBody, messageTopic); ++ } ++ ++ public MessageExtBatch buildMessageBatch(MessageBatch msgBatch) { ++ MessageExtBatch msgExtBatch = new MessageExtBatch(); ++ msgExtBatch.setTopic(messageTopic); ++ msgExtBatch.setTags("TAG1"); ++ msgExtBatch.setKeys("Hello"); ++ msgExtBatch.setBody(msgBatch.getBody()); ++ msgExtBatch.setKeys(String.valueOf(System.currentTimeMillis())); ++ msgExtBatch.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); ++ msgExtBatch.setSysFlag(0); ++ msgExtBatch.setBornTimestamp(System.currentTimeMillis()); ++ msgExtBatch.setStoreHost(storeHost); ++ msgExtBatch.setBornHost(bornHost); ++ return msgExtBatch; ++ } ++ ++ @Test ++ public void testGroupCommit() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ long totalMsgs = 10; ++ queueTotal = 1; ++ messageBody = storeMessage.getBytes(); ++ for (long i = 0; i < totalMsgs; i++) { ++ messageStore.putMessage(buildMessage()); ++ } ++ ++ for (long i = 0; i < totalMsgs; i++) { ++ GetMessageResult result = messageStore.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); ++ assertThat(result).isNotNull(); ++ result.release(); ++ } ++ verifyThatMasterIsFunctional(totalMsgs, messageStore); ++ } ++ ++ @Test ++ public void testMaxOffset() throws InterruptedException { ++ if (notExecuted()) { ++ return; ++ } ++ int firstBatchMessages = 3; ++ int queueId = 0; ++ messageBody = storeMessage.getBytes(); ++ ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(0); ++ ++ for (int i = 0; i < firstBatchMessages; i++) { ++ final MessageExtBrokerInner msg = buildMessage(); ++ msg.setQueueId(queueId); ++ messageStore.putMessage(msg); ++ } ++ ++ while (messageStore.dispatchBehindBytes() != 0) { ++ TimeUnit.MILLISECONDS.sleep(1); ++ } ++ ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages); ++ ++ // Disable the dispatcher ++ messageStore.getDispatcherList().clear(); ++ ++ int secondBatchMessages = 2; ++ ++ for (int i = 0; i < secondBatchMessages; i++) { ++ final MessageExtBrokerInner msg = buildMessage(); ++ msg.setQueueId(queueId); ++ messageStore.putMessage(msg); ++ } ++ ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages); ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, true)).isEqualTo(firstBatchMessages); ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, false)).isEqualTo(firstBatchMessages + secondBatchMessages); ++ } ++ ++ private MessageExtBrokerInner buildIPv6HostMessage() { ++ return buildIPv6HostMessage(messageBody, "FooBar"); ++ } ++ ++ private void verifyThatMasterIsFunctional(long totalMsgs, MessageStore master) { ++ for (long i = 0; i < totalMsgs; i++) { ++ master.putMessage(buildMessage()); ++ } ++ ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ for (long i = 0; i < totalMsgs; i++) { ++ GetMessageResult result = master.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); ++ assertThat(result).isNotNull(); ++ result.release(); ++ ++ } ++ } ++ ++ @Test ++ public void testPullSize() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ String topic = "pullSizeTopic"; ++ ++ for (int i = 0; i < 32; i++) { ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ // wait for consume queue build ++ // the sleep time should be great than consume queue flush interval ++ //Thread.sleep(100); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ String group = "simple"; ++ GetMessageResult getMessageResult32 = messageStore.getMessage(group, topic, 0, 0, 32, null); ++ assertThat(getMessageResult32.getMessageBufferList().size()).isEqualTo(32); ++ getMessageResult32.release(); ++ ++ GetMessageResult getMessageResult20 = messageStore.getMessage(group, topic, 0, 0, 20, null); ++ assertThat(getMessageResult20.getMessageBufferList().size()).isEqualTo(20); ++ ++ getMessageResult20.release(); ++ GetMessageResult getMessageResult45 = messageStore.getMessage(group, topic, 0, 0, 10, null); ++ assertThat(getMessageResult45.getMessageBufferList().size()).isEqualTo(10); ++ getMessageResult45.release(); ++ ++ } ++ ++ @Test ++ public void testRecover() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ String topic = "recoverTopic"; ++ messageBody = storeMessage.getBytes(); ++ for (int i = 0; i < 100; i++) { ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ ++ // Thread.sleep(100);//wait for build consumer queue ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ long maxPhyOffset = messageStore.getMaxPhyOffset(); ++ long maxCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); ++ ++ //1.just reboot ++ messageStore.shutdown(); ++ String storeRootDir = ((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir(); ++ messageStore = buildMessageStore(storeRootDir, topic); ++ boolean load = messageStore.load(); ++ assertTrue(load); ++ messageStore.start(); ++ assertTrue(maxPhyOffset == messageStore.getMaxPhyOffset()); ++ assertTrue(maxCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); ++ ++ //2.damage commit-log and reboot normal ++ for (int i = 0; i < 100; i++) { ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ //Thread.sleep(100); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ long secondLastPhyOffset = messageStore.getMaxPhyOffset(); ++ long secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); ++ ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ ++ ++ messageStore.shutdown(); ++ ++ //damage last message ++ damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset); ++ ++ //reboot ++ messageStore = buildMessageStore(storeRootDir, topic); ++ load = messageStore.load(); ++ assertTrue(load); ++ messageStore.start(); ++ assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset()); ++ assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); ++ ++ //3.damage commitlog and reboot abnormal ++ for (int i = 0; i < 100; i++) { ++ messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ //Thread.sleep(100); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ secondLastPhyOffset = messageStore.getMaxPhyOffset(); ++ secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); ++ ++ messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ messageStore.shutdown(); ++ ++ //damage last message ++ damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset); ++ //add abort file ++ String fileName = StorePathConfigHelper.getAbortFile(((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir()); ++ File file = new File(fileName); ++ UtilAll.ensureDirOK(file.getParent()); ++ file.createNewFile(); ++ ++ messageStore = buildMessageStore(storeRootDir, topic); ++ load = messageStore.load(); ++ assertTrue(load); ++ messageStore.start(); ++ assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset()); ++ assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); ++ ++ //message write again ++ for (int i = 0; i < 100; i++) { ++ messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ } ++ ++ @Test ++ public void testStorePathOK() { ++ if (notExecuted()) { ++ return; ++ } ++ if (messageStore instanceof RocksDBMessageStore) { ++ assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathPhysic())); ++ assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathLogic())); ++ } ++ } ++ ++ private boolean fileExists(String path) { ++ if (path != null) { ++ File f = new File(path); ++ return f.exists(); ++ } ++ return false; ++ } ++ ++ private void damageCommitLog(RocksDBMessageStore store, long offset) throws Exception { ++ assertThat(store).isNotNull(); ++ MessageStoreConfig messageStoreConfig = store.getMessageStoreConfig(); ++ File file = new File(messageStoreConfig.getStorePathCommitLog() + File.separator + "00000000000000000000"); ++ try (RandomAccessFile raf = new RandomAccessFile(file, "rw"); ++ FileChannel fileChannel = raf.getChannel()) { ++ MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, 1024 * 1024 * 10); ++ int bodyLen = mappedByteBuffer.getInt((int) offset + 84); ++ int topicLenIndex = (int) offset + 84 + bodyLen + 4; ++ mappedByteBuffer.position(topicLenIndex); ++ mappedByteBuffer.putInt(0); ++ mappedByteBuffer.putInt(0); ++ mappedByteBuffer.putInt(0); ++ mappedByteBuffer.putInt(0); ++ mappedByteBuffer.force(); ++ fileChannel.force(true); ++ } ++ } ++ ++ @Test ++ public void testPutMsgExceedsMaxLength() { ++ if (notExecuted()) { ++ return; ++ } ++ messageBody = new byte[4 * 1024 * 1024 + 1]; ++ MessageExtBrokerInner msg = buildMessage(); ++ ++ PutMessageResult result = messageStore.putMessage(msg); ++ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.MESSAGE_ILLEGAL); ++ } ++ ++ @Test ++ public void testPutMsgBatchExceedsMaxLength() { ++ if (notExecuted()) { ++ return; ++ } ++ messageBody = new byte[4 * 1024 * 1024 + 1]; ++ MessageExtBrokerInner msg1 = buildMessage(); ++ MessageExtBrokerInner msg2 = buildMessage(); ++ MessageExtBrokerInner msg3 = buildMessage(); ++ ++ MessageBatch msgBatch = MessageBatch.generateFromList(Arrays.asList(msg1, msg2, msg3)); ++ msgBatch.setBody(msgBatch.encode()); ++ ++ MessageExtBatch msgExtBatch = buildMessageBatch(msgBatch); ++ ++ try { ++ PutMessageResult result = this.messageStore.putMessages(msgExtBatch); ++ } catch (Exception e) { ++ assertThat(e.getMessage()).contains("message body size exceeded"); ++ } ++ } ++ ++ @Test ++ public void testPutMsgWhenReplicasNotEnough() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig(); ++ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER); ++ messageStoreConfig.setTotalReplicas(2); ++ messageStoreConfig.setInSyncReplicas(2); ++ messageStoreConfig.setEnableAutoInSyncReplicas(false); ++ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true); ++ this.messageStore.setAliveReplicaNumInGroup(1); ++ ++ MessageExtBrokerInner msg = buildMessage(); ++ PutMessageResult result = this.messageStore.putMessage(msg); ++ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH); ++ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false); ++ } ++ ++ @Test ++ public void testPutMsgWhenAdaptiveDegradation() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig(); ++ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER); ++ messageStoreConfig.setTotalReplicas(2); ++ messageStoreConfig.setInSyncReplicas(2); ++ messageStoreConfig.setEnableAutoInSyncReplicas(true); ++ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true); ++ this.messageStore.setAliveReplicaNumInGroup(1); ++ ++ MessageExtBrokerInner msg = buildMessage(); ++ PutMessageResult result = this.messageStore.putMessage(msg); ++ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); ++ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false); ++ messageStoreConfig.setEnableAutoInSyncReplicas(false); ++ } ++ ++ @Test ++ public void testGetBulkCommitLogData() { ++ if (notExecuted()) { ++ return; ++ } ++ RocksDBMessageStore defaultMessageStore = (RocksDBMessageStore) messageStore; ++ ++ messageBody = new byte[2 * 1024 * 1024]; ++ ++ for (int i = 0; i < 10; i++) { ++ MessageExtBrokerInner msg1 = buildMessage(); ++ messageStore.putMessage(msg1); ++ } ++ ++ List bufferResultList = defaultMessageStore.getBulkCommitLogData(0, (int) defaultMessageStore.getMaxPhyOffset()); ++ List msgList = new ArrayList<>(); ++ for (SelectMappedBufferResult bufferResult : bufferResultList) { ++ msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false)); ++ bufferResult.release(); ++ } ++ ++ assertThat(msgList.size()).isEqualTo(10); ++ } ++ ++ @Test ++ public void testPutLongMessage() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ CommitLog commitLog = ((RocksDBMessageStore) messageStore).getCommitLog(); ++ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig(); ++ MessageExtEncoder.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get(); ++ ++ //body size, topic size, properties size exactly equal to max size ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); ++ messageExtBrokerInner.setTopic(new String(new byte[127])); ++ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE])); ++ PutMessageResult encodeResult1 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult1 == null); ++ ++ //body size exactly more than max message body size ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]); ++ PutMessageResult encodeResult2 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult2.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ ++ //body size exactly equal to max message size ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 64 * 1024]); ++ PutMessageResult encodeResult3 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult3.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ ++ //message properties length more than properties maxSize ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); ++ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE + 1])); ++ PutMessageResult encodeResult4 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult4.getPutMessageStatus() == PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); ++ ++ //message length more than buffer length capacity ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); ++ messageExtBrokerInner.setTopic(new String(new byte[Short.MAX_VALUE])); ++ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE])); ++ PutMessageResult encodeResult5 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult5.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ } ++ ++ @Test ++ public void testDynamicMaxMessageSize() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig(); ++ int originMaxMessageSize = messageStoreConfig.getMaxMessageSize(); ++ ++ messageExtBrokerInner.setBody(new byte[originMaxMessageSize + 10]); ++ PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner); ++ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ ++ int newMaxMessageSize = originMaxMessageSize + 10; ++ messageStoreConfig.setMaxMessageSize(newMaxMessageSize); ++ putMessageResult = messageStore.putMessage(messageExtBrokerInner); ++ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK); ++ ++ messageStoreConfig.setMaxMessageSize(10); ++ putMessageResult = messageStore.putMessage(messageExtBrokerInner); ++ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ ++ messageStoreConfig.setMaxMessageSize(originMaxMessageSize); ++ } ++ ++ @Test ++ public void testDeleteTopics() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig(); ++ ConcurrentMap> consumeQueueTable = ++ ((RocksDBMessageStore) messageStore).getConsumeQueueTable(); ++ for (int i = 0; i < 10; i++) { ++ ConcurrentMap cqTable = new ConcurrentHashMap<>(); ++ String topicName = "topic-" + i; ++ for (int j = 0; j < 4; j++) { ++ ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(), ++ messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore); ++ cqTable.put(j, consumeQueue); ++ } ++ consumeQueueTable.put(topicName, cqTable); ++ } ++ Assert.assertEquals(consumeQueueTable.size(), 10); ++ HashSet resultSet = Sets.newHashSet("topic-3", "topic-5"); ++ messageStore.deleteTopics(Sets.difference(consumeQueueTable.keySet(), resultSet)); ++ Assert.assertEquals(consumeQueueTable.size(), 2); ++ Assert.assertEquals(resultSet, consumeQueueTable.keySet()); ++ } ++ ++ @Test ++ public void testCleanUnusedTopic() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig(); ++ ConcurrentMap> consumeQueueTable = ++ ((RocksDBMessageStore) messageStore).getConsumeQueueTable(); ++ for (int i = 0; i < 10; i++) { ++ ConcurrentMap cqTable = new ConcurrentHashMap<>(); ++ String topicName = "topic-" + i; ++ for (int j = 0; j < 4; j++) { ++ ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(), ++ messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore); ++ cqTable.put(j, consumeQueue); ++ } ++ consumeQueueTable.put(topicName, cqTable); ++ } ++ Assert.assertEquals(consumeQueueTable.size(), 10); ++ HashSet resultSet = Sets.newHashSet("topic-3", "topic-5"); ++ messageStore.cleanUnusedTopic(resultSet); ++ Assert.assertEquals(consumeQueueTable.size(), 2); ++ Assert.assertEquals(resultSet, consumeQueueTable.keySet()); ++ } ++ ++ private class MyMessageArrivingListener implements MessageArrivingListener { ++ @Override ++ public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, ++ byte[] filterBitMap, Map properties) { ++ } ++ } ++ ++ private boolean notExecuted() { ++ return MixAll.isMac(); ++ } ++} ++ ++ +diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java +index b2d99c3ed..17a2b5e19 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java ++++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java +@@ -38,11 +38,16 @@ public class StoreTestUtil { + + public static boolean isCommitLogAvailable(DefaultMessageStore store) { + try { ++ Field serviceField = null; ++ if (store instanceof RocksDBMessageStore) { ++ serviceField = store.getClass().getSuperclass().getDeclaredField("reputMessageService"); ++ } else { ++ serviceField = store.getClass().getDeclaredField("reputMessageService"); ++ } + +- Field serviceField = store.getClass().getDeclaredField("reputMessageService"); + serviceField.setAccessible(true); + DefaultMessageStore.ReputMessageService reputService = +- (DefaultMessageStore.ReputMessageService) serviceField.get(store); ++ (DefaultMessageStore.ReputMessageService) serviceField.get(store); + + Method method = DefaultMessageStore.ReputMessageService.class.getDeclaredMethod("isCommitLogAvailable"); + method.setAccessible(true); +diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java +index 54174ac16..fa8f41dbf 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java ++++ b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java +@@ -36,7 +36,7 @@ import org.junit.Before; + import org.junit.Test; + import org.junit.runner.RunWith; + import org.mockito.junit.MockitoJUnitRunner; +- ++import org.rocksdb.RocksDBException; + import static org.assertj.core.api.Assertions.assertThat; + import static org.awaitility.Awaitility.await; + import static org.mockito.ArgumentMatchers.anyLong; +@@ -114,7 +114,7 @@ public class HAServerTest { + } + + @Test +- public void inSyncReplicasNums() throws IOException { ++ public void inSyncReplicasNums() throws IOException, RocksDBException { + DefaultMessageStore messageStore = mockMessageStore(); + doReturn(123L).when(messageStore).getMaxPhyOffset(); + doReturn(123L).when(messageStore).getMasterFlushedOffset(); +@@ -150,7 +150,7 @@ public class HAServerTest { + } + + @Test +- public void isSlaveOK() throws IOException { ++ public void isSlaveOK() throws IOException, RocksDBException { + DefaultMessageStore messageStore = mockMessageStore(); + doReturn(123L).when(messageStore).getMaxPhyOffset(); + doReturn(123L).when(messageStore).getMasterFlushedOffset(); +@@ -175,7 +175,8 @@ public class HAServerTest { + } + + @Test +- public void putRequest_SingleAck() throws IOException, ExecutionException, InterruptedException, TimeoutException { ++ public void putRequest_SingleAck() ++ throws IOException, ExecutionException, InterruptedException, TimeoutException, RocksDBException { + CommitLog.GroupCommitRequest request = new CommitLog.GroupCommitRequest(124, 4000, 1); + this.haService.putRequest(request); + +@@ -192,7 +193,8 @@ public class HAServerTest { + } + + @Test +- public void putRequest_MultipleAckAndRequests() throws IOException, ExecutionException, InterruptedException { ++ public void putRequest_MultipleAckAndRequests() ++ throws IOException, ExecutionException, InterruptedException, RocksDBException { + CommitLog.GroupCommitRequest oneAck = new CommitLog.GroupCommitRequest(124, 4000, 2); + this.haService.putRequest(oneAck); + +@@ -218,7 +220,7 @@ public class HAServerTest { + } + + @Test +- public void getPush2SlaveMaxOffset() throws IOException { ++ public void getPush2SlaveMaxOffset() throws IOException, RocksDBException { + DefaultMessageStore messageStore = mockMessageStore(); + doReturn(123L).when(messageStore).getMaxPhyOffset(); + doReturn(123L).when(messageStore).getMasterFlushedOffset(); +@@ -256,7 +258,7 @@ public class HAServerTest { + this.haClientList.add(haClient); + } + +- private DefaultMessageStore mockMessageStore() throws IOException { ++ private DefaultMessageStore mockMessageStore() throws IOException, RocksDBException { + DefaultMessageStore messageStore = mock(DefaultMessageStore.class); + BrokerConfig brokerConfig = mock(BrokerConfig.class); + +diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java +index 27dcff141..db5c5af4c 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java ++++ b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java +@@ -41,6 +41,7 @@ import org.junit.Assert; + import org.junit.Assume; + import org.junit.Ignore; + import org.junit.Test; ++import org.rocksdb.RocksDBException; + + import java.io.File; + import java.net.InetAddress; +@@ -180,7 +181,7 @@ public class AutoSwitchHATest { + + private boolean changeMasterAndPutMessage(DefaultMessageStore master, MessageStoreConfig masterConfig, + DefaultMessageStore slave, long slaveId, MessageStoreConfig slaveConfig, int epoch, String masterHaAddress, +- int totalPutMessageNums) { ++ int totalPutMessageNums) throws RocksDBException { + + boolean flag = true; + // Change role +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +index d7d13d61e..edaa5d19f 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +@@ -16,17 +16,14 @@ + */ + package org.apache.rocketmq.tieredstore; + +-import com.google.common.base.Stopwatch; +-import io.opentelemetry.api.common.Attributes; +-import io.opentelemetry.api.common.AttributesBuilder; +-import io.opentelemetry.api.metrics.Meter; +-import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.util.List; + import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.TimeUnit; + import java.util.function.Supplier; ++ ++import com.google.common.base.Stopwatch; ++ + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.MixAll; +@@ -55,6 +52,12 @@ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant; + import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; + import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + ++import io.opentelemetry.api.common.Attributes; ++import io.opentelemetry.api.common.AttributesBuilder; ++import io.opentelemetry.api.metrics.Meter; ++import io.opentelemetry.sdk.metrics.InstrumentSelector; ++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ + public class TieredMessageStore extends AbstractPluginMessageStore { + + protected static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +index 2a7d3fba4..1ecb1fa2c 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +@@ -106,8 +106,8 @@ public class ExportMetadataInRocksDBCommand implements SubCommand { + final Map jsonConfig = new HashMap<>(); + final Map configTable = new HashMap<>(); + iterateKvStore(kvStore, (key, value) -> { +- final String configKey = new String(key, DataConverter.charset); +- final String configValue = new String(value, DataConverter.charset); ++ final String configKey = new String(key, DataConverter.CHARSET_UTF8); ++ final String configValue = new String(value, DataConverter.CHARSET_UTF8); + final JSONObject jsonObject = JSONObject.parseObject(configValue); + configTable.put(configKey, jsonObject); + } +@@ -120,8 +120,8 @@ public class ExportMetadataInRocksDBCommand implements SubCommand { + } else { + AtomicLong count = new AtomicLong(0); + iterateKvStore(kvStore, (key, value) -> { +- final String configKey = new String(key, DataConverter.charset); +- final String configValue = new String(value, DataConverter.charset); ++ final String configKey = new String(key, DataConverter.CHARSET_UTF8); ++ final String configValue = new String(value, DataConverter.CHARSET_UTF8); + System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), configKey, configValue); + }); + } +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +new file mode 100644 +index 000000000..b987ad873 +--- /dev/null ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +@@ -0,0 +1,118 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.tools.command.metadata; ++ ++import com.alibaba.fastjson.JSONObject; ++import org.apache.commons.cli.CommandLine; ++import org.apache.commons.cli.Option; ++import org.apache.commons.cli.Options; ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.config.RocksDBConfigManager; ++import org.apache.rocketmq.common.utils.DataConverter; ++import org.apache.rocketmq.remoting.RPCHook; ++import org.apache.rocketmq.tools.command.SubCommand; ++import org.apache.rocketmq.tools.command.SubCommandException; ++ ++import java.io.File; ++import java.util.HashMap; ++import java.util.Map; ++ ++public class RocksDBConfigToJsonCommand implements SubCommand { ++ private static final String TOPICS_JSON_CONFIG = "topics"; ++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; ++ ++ @Override ++ public String commandName() { ++ return "rocksDBConfigToJson"; ++ } ++ ++ @Override ++ public String commandDesc() { ++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; ++ } ++ ++ @Override ++ public Options buildCommandlineOptions(Options options) { ++ Option pathOption = new Option("p", "path", true, ++ "Absolute path to the metadata directory"); ++ pathOption.setRequired(true); ++ options.addOption(pathOption); ++ ++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + ++ "topics/subscriptionGroups"); ++ configTypeOption.setRequired(true); ++ options.addOption(configTypeOption); ++ ++ return options; ++ } ++ ++ @Override ++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { ++ String path = commandLine.getOptionValue("path").trim(); ++ if (StringUtils.isEmpty(path) || !new File(path).exists()) { ++ System.out.print("Rocksdb path is invalid.\n"); ++ return; ++ } ++ ++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); ++ ++ final long memTableFlushInterval = 60 * 60 * 1000L; ++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); ++ try { ++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for topics.json ++ final Map topicsJsonConfig = new HashMap<>(); ++ final Map topicConfigTable = new HashMap<>(); ++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++ final String topic = new String(key, DataConverter.CHARSET_UTF8); ++ final String topicConfig = new String(value, DataConverter.CHARSET_UTF8); ++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig); ++ topicConfigTable.put(topic, jsonObject); ++ }); ++ ++ if (isLoad) { ++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); ++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); ++ System.out.print(topicsJsonStr + "\n"); ++ return; ++ } ++ } ++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for subscriptionGroup.json ++ final Map subscriptionGroupJsonConfig = new HashMap<>(); ++ final Map subscriptionGroupTable = new HashMap<>(); ++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++ final String subscriptionGroup = new String(key, DataConverter.CHARSET_UTF8); ++ final String subscriptionGroupConfig = new String(value, DataConverter.CHARSET_UTF8); ++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); ++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); ++ }); ++ ++ if (isLoad) { ++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", ++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); ++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); ++ System.out.print(subscriptionGroupJsonStr + "\n"); ++ return; ++ } ++ } ++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); ++ } finally { ++ kvConfigManager.stop(); ++ } ++ } ++} +-- +2.32.0.windows.2 + diff --git a/rocketmq.spec b/rocketmq.spec index f67608c..ed62612 100644 --- a/rocketmq.spec +++ b/rocketmq.spec @@ -5,7 +5,7 @@ Summary: Cloud-Native, Distributed Messaging and Streaming Name: rocketmq Version: 5.1.3 -Release: 22 +Release: 23 License: Apache-2.0 Group: Applications/Message URL: https://rocketmq.apache.org/ @@ -31,6 +31,7 @@ Patch0018: patch018-backport-enhancement-of-tiered-storage.patch Patch0019: patch019-backport-some-bugfix.patch Patch0020: patch020-backport-add-goaway-mechanism.patch Patch0021: patch021-backport-some-enhancements.patch +Patch0022: patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch BuildRequires: java-1.8.0-openjdk-devel, maven, maven-local, git Requires: java-1.8.0-openjdk-devel @@ -65,6 +66,9 @@ exit 0 %changelog +* Fri Dec 8 2023 ShiZhili - 5.1.3-23 +- backport support kv storage + * Tue Dec 5 2023 ShiZhili - 5.1.3-22 - backport some enhancements