From 0a046b85611bdc93c529650dc951f4cabbb61db9 Mon Sep 17 00:00:00 2001 From: zhiliatom Date: Fri, 8 Dec 2023 09:25:41 +0800 Subject: [PATCH] [ISSUE #7064] [RIP-66-2] Support KV(RocksDB) Storage for ConsumeQueue --- .../rocketmq/broker/BrokerController.java | 74 +- .../broker/controller/ReplicasManager.java | 4 +- .../offset/RocksDBConsumerOffsetManager.java | 8 +- .../broker/processor/AckMessageProcessor.java | 2 +- .../processor/AdminBrokerProcessor.java | 28 +- .../ChangeInvisibleTimeProcessor.java | 4 +- .../processor/PopBufferMergeService.java | 4 +- .../broker/processor/PopMessageProcessor.java | 2 +- .../broker/processor/PopReviveService.java | 6 +- .../RocksDBSubscriptionGroupManager.java | 10 +- .../topic/RocksDBTopicConfigManager.java | 8 +- .../processor/PopReviveServiceTest.java | 4 +- common/pom.xml | 2 +- .../org/apache/rocketmq/common/MixAll.java | 1 + .../rocketmq/common/attribute/CQType.java | 3 +- .../common/config/AbstractRocksDBStorage.java | 112 +- .../common/config/ConfigRocksDBStorage.java | 2 +- .../common/message/MessageExtBrokerInner.java | 13 + .../rocketmq/common/topic/TopicValidator.java | 2 + .../rocketmq/common/utils/DataConverter.java | 2 +- .../rocketmq/example/quickstart/Producer.java | 2 +- metrics | 241 ++ patch009-modify-CRLF-to-LF.patch | 127 + patch010-backport-add-some-fixes | 1286 ++++++ patch011-backport-optimize-config | 1390 +++++++ ...optimize-opentelemetry-metric-config.patch | 2081 ++++++++++ ...-backport-enhance-rockdbconfigtojson.patch | 2920 +++++++++++++ patch013-backport-enhance-admin-output.patch | 892 ++++ ...ueue-Selection-Strategy-Optimization.patch | 2023 +++++++++ patch015-backport-fix-some-bugs.patch | 1894 +++++++++ ...rt-Optimize-fault-tolerant-mechanism.patch | 520 +++ ...port-Convergent-thread-pool-creation.patch | 2243 ++++++++++ ...ckport-enhancement-of-tiered-storage.patch | 601 +++ patch019-backport-some-bugfix.patch | 1499 +++++++ patch020-backport-add-goaway-mechanism.patch | 3696 +++++++++++++++++ patch021-backport-some-enhancements.patch | 344 ++ pom.xml | 4 +- store/pom.xml | 4 + .../org/apache/rocketmq/store/CommitLog.java | 94 +- .../rocketmq/store/CommitLogDispatcher.java | 5 +- .../apache/rocketmq/store/ConsumeQueue.java | 93 +- .../rocketmq/store/DefaultMessageStore.java | 279 +- .../apache/rocketmq/store/MessageStore.java | 54 +- .../rocketmq/store/RocksDBMessageStore.java | 169 + .../apache/rocketmq/store/RunningFlags.java | 22 +- .../store/config/MessageStoreConfig.java | 33 +- .../store/dledger/DLedgerCommitLog.java | 53 +- .../apache/rocketmq/store/ha/HAService.java | 3 +- .../ha/autoswitch/AutoSwitchHAClient.java | 2 +- .../ha/autoswitch/AutoSwitchHAService.java | 7 +- .../plugin/AbstractPluginMessageStore.java | 42 +- .../queue/AbstractConsumeQueueStore.java | 105 + .../store/queue/BatchConsumeQueue.java | 20 + .../store/queue/ConsumeQueueInterface.java | 27 +- .../store/queue/ConsumeQueueStore.java | 293 +- .../queue/ConsumeQueueStoreInterface.java | 289 ++ .../rocketmq/store/queue/MultiDispatch.java | 76 + .../store/queue/QueueOffsetOperator.java | 8 + .../store/queue/RocksDBConsumeQueue.java | 437 ++ .../queue/RocksDBConsumeQueueOffsetTable.java | 641 +++ .../store/queue/RocksDBConsumeQueueStore.java | 441 ++ .../store/queue/RocksDBConsumeQueueTable.java | 312 ++ .../ConsumeQueueCompactionFilterFactory.java | 47 + .../rocksdb/ConsumeQueueRocksDBStorage.java | 133 + .../store/rocksdb/RocksDBOptionsFactory.java | 161 + .../store/timer/TimerMessageStore.java | 65 +- .../store/DefaultMessageStoreTest.java | 5 +- .../rocketmq/store/MultiDispatchTest.java | 8 +- .../store/RocksDBMessageStoreTest.java | 1060 +++++ .../apache/rocketmq/store/StoreTestUtil.java | 9 +- .../rocketmq/store/ha/HAServerTest.java | 16 +- .../store/ha/autoswitch/AutoSwitchHATest.java | 3 +- .../tieredstore/TieredMessageStore.java | 15 +- .../ExportMetadataInRocksDBCommand.java | 8 +- .../metadata/RocksDBConfigToJsonCommand.java | 118 + 75 files changed, 26543 insertions(+), 668 deletions(-) create mode 100644 metrics create mode 100644 patch009-modify-CRLF-to-LF.patch create mode 100644 patch010-backport-add-some-fixes create mode 100644 patch011-backport-optimize-config create mode 100644 patch011-backport-optimize-opentelemetry-metric-config.patch create mode 100644 patch012-backport-enhance-rockdbconfigtojson.patch create mode 100644 patch013-backport-enhance-admin-output.patch create mode 100644 patch014-backport-Queue-Selection-Strategy-Optimization.patch create mode 100644 patch015-backport-fix-some-bugs.patch create mode 100644 patch016-backport-Optimize-fault-tolerant-mechanism.patch create mode 100644 patch017-backport-Convergent-thread-pool-creation.patch create mode 100644 patch018-backport-enhancement-of-tiered-storage.patch create mode 100644 patch019-backport-some-bugfix.patch create mode 100644 patch020-backport-add-goaway-mechanism.patch create mode 100644 patch021-backport-some-enhancements.patch create mode 100644 store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java create mode 100644 store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index d4bded600..9f1fd0ad0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -16,7 +16,33 @@ */ package org.apache.rocketmq.broker; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.stream.Collectors; + import com.google.common.collect.Lists; + import org.apache.rocketmq.acl.AccessValidator; import org.apache.rocketmq.acl.plain.PlainAccessValidator; import org.apache.rocketmq.broker.client.ClientHousekeepingService; @@ -126,7 +152,7 @@ import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageArrivingListener; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; -import org.apache.rocketmq.store.StoreType; +import org.apache.rocketmq.store.RocksDBMessageStore; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.dledger.DLedgerCommitLog; @@ -141,31 +167,6 @@ import org.apache.rocketmq.store.timer.TimerCheckpoint; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.timer.TimerMetrics; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; -import java.util.stream.Collectors; - public class BrokerController { protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); @@ -308,7 +309,7 @@ public class BrokerController { this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort())); this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()); this.broadcastOffsetManager = new BroadcastOffsetManager(this); - if (isEnableRocksDBStore()) { + if (this.messageStoreConfig.isEnableRocksDBStore()) { this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this); this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this); this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqConsumerOffsetManager(this) : new RocksDBConsumerOffsetManager(this); @@ -747,7 +748,12 @@ public class BrokerController { public boolean initializeMessageStore() { boolean result = true; try { - DefaultMessageStore defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); + DefaultMessageStore defaultMessageStore; + if (this.messageStoreConfig.isEnableRocksDBStore()) { + defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); + } else { + defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); + } if (messageStoreConfig.isEnableDLegerCommitLog()) { DLedgerRoleChangeHandler roleChangeHandler = @@ -944,16 +950,16 @@ public class BrokerController { this.transactionalMessageService = ServiceProvider.loadClass(TransactionalMessageService.class); if (null == this.transactionalMessageService) { this.transactionalMessageService = new TransactionalMessageServiceImpl( - new TransactionalMessageBridge(this, this.getMessageStore())); + new TransactionalMessageBridge(this, this.getMessageStore())); LOG.warn("Load default transaction message hook service: {}", - TransactionalMessageServiceImpl.class.getSimpleName()); + TransactionalMessageServiceImpl.class.getSimpleName()); } this.transactionalMessageCheckListener = ServiceProvider.loadClass( - AbstractTransactionalMessageCheckListener.class); + AbstractTransactionalMessageCheckListener.class); if (null == this.transactionalMessageCheckListener) { this.transactionalMessageCheckListener = new DefaultTransactionalMessageCheckListener(); LOG.warn("Load default discard message hook service: {}", - DefaultTransactionalMessageCheckListener.class.getSimpleName()); + DefaultTransactionalMessageCheckListener.class.getSimpleName()); } this.transactionalMessageCheckListener.setBrokerController(this); this.transactionalMessageCheckService = new TransactionalMessageCheckService(this); @@ -2412,8 +2418,4 @@ public class BrokerController { public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) { this.coldDataCgCtrService = coldDataCgCtrService; } - - public boolean isEnableRocksDBStore() { - return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.messageStoreConfig.getStoreType()); - } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java index a989e6e68..a1d711cb2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java @@ -224,7 +224,7 @@ public class ReplicasManager { public synchronized void changeBrokerRole(final Long newMasterBrokerId, final String newMasterAddress, final Integer newMasterEpoch, - final Integer syncStateSetEpoch, final Set syncStateSet) { + final Integer syncStateSetEpoch, final Set syncStateSet) throws Exception { if (newMasterBrokerId != null && newMasterEpoch > this.masterEpoch) { if (newMasterBrokerId.equals(this.brokerControllerId)) { changeToMaster(newMasterEpoch, syncStateSetEpoch, syncStateSet); @@ -234,7 +234,7 @@ public class ReplicasManager { } } - public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set syncStateSet) { + public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set syncStateSet) throws Exception { synchronized (this) { if (newMasterEpoch > this.masterEpoch) { LOGGER.info("Begin to change to master, brokerName:{}, replicas:{}, new Epoch:{}", this.brokerConfig.getBrokerName(), this.brokerAddress, newMasterEpoch); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java index 5695a3356..05b53b0bc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java @@ -33,7 +33,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { public RocksDBConsumerOffsetManager(BrokerController brokerController) { super(brokerController); - this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); + this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override @@ -49,7 +49,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { @Override protected void removeConsumerOffset(String topicAtGroup) { try { - byte[] keyBytes = topicAtGroup.getBytes(DataConverter.charset); + byte[] keyBytes = topicAtGroup.getBytes(DataConverter.CHARSET_UTF8); this.rocksDBConfigManager.delete(keyBytes); } catch (Exception e) { LOG.error("kv remove consumerOffset Failed, {}", topicAtGroup); @@ -58,7 +58,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { @Override protected void decode0(final byte[] key, final byte[] body) { - String topicAtGroup = new String(key, DataConverter.charset); + String topicAtGroup = new String(key, DataConverter.CHARSET_UTF8); RocksDBOffsetSerializeWrapper wrapper = JSON.parseObject(body, RocksDBOffsetSerializeWrapper.class); this.offsetTable.put(topicAtGroup, wrapper.getOffsetTable()); @@ -93,7 +93,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { } private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupName, final ConcurrentMap offsetMap) throws Exception { - byte[] keyBytes = topicGroupName.getBytes(DataConverter.charset); + byte[] keyBytes = topicGroupName.getBytes(DataConverter.CHARSET_UTF8); RocksDBOffsetSerializeWrapper wrapper = new RocksDBOffsetSerializeWrapper(); wrapper.setOffsetTable(offsetMap); byte[] valueBytes = JSON.toJSONBytes(wrapper, SerializerFeature.BrowserCompatible); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java index 244b459d6..59a3e63b2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java @@ -253,7 +253,7 @@ public class AckMessageProcessor implements NettyRequestProcessor { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(rqId); if (ackMsg instanceof BatchAckMsg) { msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index e77120e15..dd4ec960f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -539,14 +539,18 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic); // delete pop retry topics first - for (String group : groups) { - final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); - if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { - deleteTopicInBroker(popRetryTopic); + try { + for (String group : groups) { + final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); + if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { + deleteTopicInBroker(popRetryTopic); + } } + // delete topic + deleteTopicInBroker(topic); + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); } - // delete topic - deleteTopicInBroker(topic); response.setCode(ResponseCode.SUCCESS); response.setRemark(null); return response; @@ -2081,7 +2085,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { public RemotingCommand cleanExpiredConsumeQueue() { LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: start."); final RemotingCommand response = RemotingCommand.createResponseCommand(null); - brokerController.getMessageStore().cleanExpiredConsumerQueue(); + try { + brokerController.getMessageStore().cleanExpiredConsumerQueue(); + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: end."); response.setCode(ResponseCode.SUCCESS); response.setRemark(null); @@ -2781,7 +2789,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { final ReplicasManager replicasManager = this.brokerController.getReplicasManager(); if (replicasManager != null) { - replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet()); + try { + replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet()); + } catch (Exception e) { + throw new RemotingCommandException(e.getMessage()); + } } response.setCode(ResponseCode.SUCCESS); response.setRemark(null); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java index 2ccdf07f6..bdfffff09 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java @@ -180,7 +180,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { } msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(rqId); msgInner.setTags(PopAckConstants.ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); @@ -216,7 +216,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { ck.addDiff(0); ck.setBrokerName(brokerName); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java index b7ba8ad4a..8a85dd8fe 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java @@ -633,7 +633,7 @@ public class PopBufferMergeService extends ServiceThread { ackMsg.setQueueId(point.getQueueId()); ackMsg.setPopTime(point.getPopTime()); msgInner.setTopic(popMessageProcessor.reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(pointWrapper.getReviveQueueId()); msgInner.setTags(PopAckConstants.ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); @@ -673,7 +673,7 @@ public class PopBufferMergeService extends ServiceThread { batchAckMsg.setQueueId(point.getQueueId()); batchAckMsg.setPopTime(point.getPopTime()); msgInner.setTopic(popMessageProcessor.reviveTopic); - msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(pointWrapper.getReviveQueueId()); msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 0d9bdf143..f5d07c5aa 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -685,7 +685,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index d5174d3d1..4f80752e1 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -356,7 +356,7 @@ public class PopReviveService extends ServiceThread { } for (MessageExt messageExt : messageExts) { if (PopAckConstants.CK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.charset); + String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("reviveQueueId={},find ck, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } @@ -371,7 +371,7 @@ public class PopReviveService extends ServiceThread { firstRt = point.getReviveTime(); } } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.charset); + String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("reviveQueueId={},find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } @@ -395,7 +395,7 @@ public class PopReviveService extends ServiceThread { } } } else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.charset); + String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java index 6503970af..e9a81a8d6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java @@ -30,7 +30,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { public RocksDBSubscriptionGroupManager(BrokerController brokerController) { super(brokerController, false); - this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); + this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override @@ -53,7 +53,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig); try { - byte[] keyBytes = groupName.getBytes(DataConverter.charset); + byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8); byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible); this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); } catch (Exception e) { @@ -68,7 +68,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.putIfAbsent(groupName, subscriptionGroupConfig); if (oldConfig == null) { try { - byte[] keyBytes = groupName.getBytes(DataConverter.charset); + byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8); byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible); this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); } catch (Exception e) { @@ -82,7 +82,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) { SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.remove(groupName); try { - this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.charset)); + this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.CHARSET_UTF8)); } catch (Exception e) { log.error("kv delete sub Failed, {}", subscriptionGroupConfig.toString()); } @@ -91,7 +91,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { @Override protected void decode0(byte[] key, byte[] body) { - String groupName = new String(key, DataConverter.charset); + String groupName = new String(key, DataConverter.CHARSET_UTF8); SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class); this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java index 7da0d7c8a..fddecf2d9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java @@ -30,7 +30,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { public RocksDBTopicConfigManager(BrokerController brokerController) { super(brokerController, false); - this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); + this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override @@ -49,7 +49,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { @Override protected void decode0(byte[] key, byte[] body) { - String topicName = new String(key, DataConverter.charset); + String topicName = new String(key, DataConverter.CHARSET_UTF8); TopicConfig topicConfig = JSON.parseObject(body, TopicConfig.class); this.topicConfigTable.put(topicName, topicConfig); @@ -66,7 +66,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { String topicName = topicConfig.getTopicName(); TopicConfig oldTopicConfig = this.topicConfigTable.put(topicName, topicConfig); try { - byte[] keyBytes = topicName.getBytes(DataConverter.charset); + byte[] keyBytes = topicName.getBytes(DataConverter.CHARSET_UTF8); byte[] valueBytes = JSON.toJSONBytes(topicConfig, SerializerFeature.BrowserCompatible); this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); } catch (Exception e) { @@ -79,7 +79,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { protected TopicConfig removeTopicConfig(String topicName) { TopicConfig topicConfig = this.topicConfigTable.remove(topicName); try { - this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.charset)); + this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.CHARSET_UTF8)); } catch (Exception e) { log.error("kv remove topic Failed, {}", topicConfig.toString()); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java index 1c3a0cd45..78b76264f 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java @@ -234,7 +234,7 @@ public class PopReviveServiceTest { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(REVIVE_TOPIC); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(REVIVE_QUEUE_ID); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); @@ -269,7 +269,7 @@ public class PopReviveServiceTest { SocketAddress host, long deliverMs, String ackUniqueId) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/common/pom.xml b/common/pom.xml index 6104c3ac6..a28ed228f 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -109,7 +109,7 @@ rocketmq-logback-classic - io.github.aliyunmq + org.apache.rocketmq rocketmq-rocksdb diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 1233a5422..407ef2842 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -492,6 +492,7 @@ public class MixAll { public static int compareLong(long x, long y) { return Long.compare(x, y); } + public static boolean isLmq(String lmqMetaData) { return lmqMetaData != null && lmqMetaData.startsWith(LMQ_PREFIX); } diff --git a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java index 73ef21880..9148d5a18 100644 --- a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java +++ b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java @@ -19,5 +19,6 @@ package org.apache.rocketmq.common.attribute; public enum CQType { SimpleCQ, - BatchCQ + BatchCQ, + RocksDBCQ } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index 6f19a9815..20319abba 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.common.config; import java.nio.ByteBuffer; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -27,11 +26,11 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -47,7 +46,6 @@ import org.rocksdb.Priority; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; import org.rocksdb.Statistics; import org.rocksdb.Status; import org.rocksdb.WriteBatch; @@ -58,7 +56,6 @@ import static org.rocksdb.RocksDB.NOT_FOUND; public abstract class AbstractRocksDBStorage { protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); - private static final Charset CHARSET_UTF8 = Charset.forName("UTF-8"); private static final String SPACE = " | "; protected String dbPath; @@ -223,10 +220,6 @@ public abstract class AbstractRocksDBStorage { } } - protected WrappedRocksIterator newIterator(ColumnFamilyHandle cfHandle, ReadOptions readOptions) { - return new WrappedRocksIterator(this.db.newIterator(cfHandle, readOptions)); - } - protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, final byte[] startKey, final byte[] endKey) throws RocksDBException { if (!hold()) { @@ -243,46 +236,6 @@ public abstract class AbstractRocksDBStorage { } } - protected void manualCompactionDefaultCfMaxLevel(final CompactionOptions compactionOptions) throws Exception { - final ColumnFamilyHandle defaultCFHandle = this.defaultCFHandle; - final byte[] defaultCFName = defaultCFHandle.getName(); - List fileMetaDataList = this.db.getLiveFilesMetaData(); - if (fileMetaDataList == null || fileMetaDataList.isEmpty()) { - return; - } - - List defaultLiveFileDataList = Lists.newArrayList(); - List inputFileNames = Lists.newArrayList(); - int maxLevel = 0; - for (LiveFileMetaData fileMetaData : fileMetaDataList) { - if (compareTo(fileMetaData.columnFamilyName(), defaultCFName) != 0) { - continue; - } - defaultLiveFileDataList.add(fileMetaData); - if (fileMetaData.level() > maxLevel) { - maxLevel = fileMetaData.level(); - } - } - if (maxLevel == 0) { - LOGGER.info("manualCompactionDefaultCfFiles skip level 0."); - return; - } - - for (LiveFileMetaData fileMetaData : defaultLiveFileDataList) { - if (fileMetaData.level() != maxLevel || fileMetaData.beingCompacted()) { - continue; - } - inputFileNames.add(fileMetaData.path() + fileMetaData.fileName()); - } - if (!inputFileNames.isEmpty()) { - List outputLists = this.db.compactFiles(compactionOptions, defaultCFHandle, - inputFileNames, maxLevel, -1, null); - LOGGER.info("manualCompactionDefaultCfFiles OK. src: {}, dst: {}", inputFileNames, outputLists); - } else { - LOGGER.info("manualCompactionDefaultCfFiles Empty."); - } - } - protected void manualCompactionDefaultCfRange(CompactRangeOptions compactRangeOptions) { if (!hold()) { return; @@ -494,50 +447,6 @@ public abstract class AbstractRocksDBStorage { this.db.flushWal(true); } - protected class WrappedRocksIterator { - private final RocksIterator iterator; - - public WrappedRocksIterator(final RocksIterator iterator) { - this.iterator = iterator; - } - - public byte[] key() { - return iterator.key(); - } - - public byte[] value() { - return iterator.value(); - } - - public void next() { - iterator.next(); - } - - public void prev() { - iterator.prev(); - } - - public void seek(byte[] target) { - iterator.seek(target); - } - - public void seekForPrev(byte[] target) { - iterator.seekForPrev(target); - } - - public void seekToFirst() { - iterator.seekToFirst(); - } - - public boolean isValid() { - return iterator.isValid(); - } - - public void close() { - iterator.close(); - } - } - private String getStatusError(RocksDBException e) { if (e == null || e.getStatus() == null) { return "null"; @@ -574,7 +483,7 @@ public abstract class AbstractRocksDBStorage { sb = new StringBuilder(256); map.put(metaData.level(), sb); } - sb.append(new String(metaData.columnFamilyName(), CHARSET_UTF8)).append(SPACE). + sb.append(new String(metaData.columnFamilyName(), DataConverter.CHARSET_UTF8)).append(SPACE). append(metaData.fileName()).append(SPACE). append("s: ").append(metaData.size()).append(SPACE). append("a: ").append(metaData.numEntries()).append(SPACE). @@ -595,21 +504,4 @@ public abstract class AbstractRocksDBStorage { } catch (Exception ignored) { } } - - public int compareTo(byte[] v1, byte[] v2) { - int len1 = v1.length; - int len2 = v2.length; - int lim = Math.min(len1, len2); - - int k = 0; - while (k < lim) { - byte c1 = v1[k]; - byte c2 = v2[k]; - if (c1 != c2) { - return c1 - c2; - } - k++; - } - return len1 - len2; - } } \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java index 463bd8fed..b40f8046e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java @@ -203,7 +203,7 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage { setUseDirectReads(true); } - private static String getDBLogDir() { + public static String getDBLogDir() { String rootPath = System.getProperty("user.home"); if (StringUtils.isEmpty(rootPath)) { return ""; diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java index 0c72ebb7b..91599653c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java @@ -70,4 +70,17 @@ public class MessageExtBrokerInner extends MessageExt { public void setVersion(MessageVersion version) { this.version = version; } + + public void removeWaitStorePropertyString() { + if (this.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) { + // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message. + // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it. + String waitStoreMsgOKValue = this.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); + this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties())); + // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later + this.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue); + } else { + this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties())); + } + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java index 61265c05d..c19592a44 100644 --- a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java +++ b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java @@ -31,6 +31,7 @@ public class TopicValidator { public static final String RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC = "TRANS_CHECK_MAX_TIME_TOPIC"; public static final String RMQ_SYS_SELF_TEST_TOPIC = "SELF_TEST_TOPIC"; public static final String RMQ_SYS_OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT"; + public static final String RMQ_SYS_ROCKSDB_OFFSET_TOPIC = "CHECKPOINT_TOPIC"; public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; public static final String SYNC_BROKER_MEMBER_GROUP_PREFIX = SYSTEM_TOPIC_PREFIX + "SYNC_BROKER_MEMBER_"; @@ -55,6 +56,7 @@ public class TopicValidator { SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC); SYSTEM_TOPIC_SET.add(RMQ_SYS_SELF_TEST_TOPIC); SYSTEM_TOPIC_SET.add(RMQ_SYS_OFFSET_MOVED_EVENT); + SYSTEM_TOPIC_SET.add(RMQ_SYS_ROCKSDB_OFFSET_TOPIC); NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_SCHEDULE_TOPIC); NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_TRANS_HALF_TOPIC); diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java index 8b50de12b..cc96770b2 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java @@ -20,7 +20,7 @@ import java.nio.ByteBuffer; import java.nio.charset.Charset; public class DataConverter { - public static Charset charset = Charset.forName("UTF-8"); + public static final Charset CHARSET_UTF8 = Charset.forName("UTF-8"); public static byte[] Long2Byte(Long v) { ByteBuffer tmp = ByteBuffer.allocate(8); diff --git a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java index aac295030..8662328ea 100644 --- a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java @@ -71,7 +71,7 @@ public class Producer { TAG /* Tag */, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */ ); - + msg.setDelayTimeLevel(1); /* * Call send message to deliver message to one of brokers. */ diff --git a/metrics b/metrics new file mode 100644 index 000000000..146e90fae --- /dev/null +++ b/metrics @@ -0,0 +1,241 @@ +# TYPE target info +# HELP target Target metadata +target_info{} 1 +# TYPE otel_scope_info info +# HELP otel_scope_info Scope metadata +otel_scope_info{otel_scope_name="broker-meter"} 1 +# TYPE rocketmq_storage_message_reserve_time gauge +# HELP rocketmq_storage_message_reserve_time default view for gauge. +rocketmq_storage_message_reserve_time_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1.9491132515E10 1696905287361 +# TYPE rocketmq_timer_enqueue_latency gauge +# HELP rocketmq_timer_enqueue_latency default view for gauge. +rocketmq_timer_enqueue_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 +# TYPE rocketmq_timer_enqueue_lag gauge +# HELP rocketmq_timer_enqueue_lag default view for gauge. +rocketmq_timer_enqueue_lag{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 +# TYPE rocketmq_timing_messages gauge +# HELP rocketmq_timing_messages default view for gauge. +rocketmq_timing_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local",topic="BenchmarkTest"} 0.0 1696905287361 +# TYPE rocketmq_storage_flush_behind_bytes gauge +# HELP rocketmq_storage_flush_behind_bytes default view for gauge. +rocketmq_storage_flush_behind_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1230.0 1696905287361 +# TYPE rocketmq_processor_watermark gauge +# HELP rocketmq_processor_watermark default view for gauge. +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="reply"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="transaction"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="send"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="async_put"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="lite_pull"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="pull"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="heartbeat"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="consumer_manager"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="admin"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="client_manager"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="query_message"} 0.0 1696905287361 +rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="ack"} 0.0 1696905287361 +# TYPE rocketmq_pop_buffer_scan_time_consume histogram +# HELP rocketmq_pop_buffer_scan_time_consume Time consuming of pop buffer scan +rocketmq_pop_buffer_scan_time_consume_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 5267005.0 1696905287361 +rocketmq_pop_buffer_scan_time_consume_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 39651.0 1696905287361 +rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="1.0"} 5266935.0 1696905287361 +rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="10.0"} 5266998.0 1696905287361 +rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="100.0"} 5267001.0 1696905287361 +rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="1000.0"} 5267005.0 1696905287361 +rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="2000.0"} 5267005.0 1696905287361 +rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="3000.0"} 5267005.0 1696905287361 +rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="+Inf"} 5267005.0 1696905287361 +# TYPE rocketmq_pop_revive_lag gauge +# HELP rocketmq_pop_revive_lag default view for gauge. +rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="1"} 1.0 1696905287361 +rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="2"} 1.0 1696905287361 +rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="3"} 1.0 1696905287361 +rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="4"} 1.0 1696905287361 +rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="5"} 1.0 1696905287361 +rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="6"} 1.0 1696905287361 +rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="7"} 1.0 1696905287361 +rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="0"} 1.0 1696905287361 +# TYPE rocketmq_pop_revive_latency gauge +# HELP rocketmq_pop_revive_latency default view for gauge. +rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="1"} 0.0 1696905287361 +rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="2"} 0.0 1696905287361 +rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="3"} 0.0 1696905287361 +rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="4"} 0.0 1696905287361 +rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="5"} 0.0 1696905287361 +rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="6"} 0.0 1696905287361 +rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="7"} 0.0 1696905287361 +rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="0"} 0.0 1696905287361 +# TYPE rocketmq_storage_size gauge +# HELP rocketmq_storage_size default view for gauge. +rocketmq_storage_size_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1.5905103872E10 1696905287361 +# TYPE rocketmq_pop_offset_buffer_size gauge +# HELP rocketmq_pop_offset_buffer_size default view for gauge. +rocketmq_pop_offset_buffer_size{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 0.0 1696905287361 +# TYPE rocketmq_timer_dequeue_latency gauge +# HELP rocketmq_timer_dequeue_latency default view for gauge. +rocketmq_timer_dequeue_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 +# TYPE rocketmq_broker_permission gauge +# HELP rocketmq_broker_permission default view for gauge. +rocketmq_broker_permission{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 6.0 1696905287361 +# TYPE rocketmq_consumer_lag_messages gauge +# HELP rocketmq_consumer_lag_messages default view for gauge. +rocketmq_consumer_lag_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 +# TYPE rocketmq_throughput_out_total counter +# HELP rocketmq_throughput_out_total default view for counter. +rocketmq_throughput_out_total{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361 +# TYPE rocketmq_timer_dequeue_lag gauge +# HELP rocketmq_timer_dequeue_lag default view for gauge. +rocketmq_timer_dequeue_lag{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 +# TYPE rocketmq_messages_out_total counter +# HELP rocketmq_messages_out_total default view for counter. +rocketmq_messages_out_total{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361 +# TYPE rocketmq_messages_in_total counter +# HELP rocketmq_messages_in_total default view for counter. +rocketmq_messages_in_total{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361 +# TYPE rocketmq_consumer_queueing_latency gauge +# HELP rocketmq_consumer_queueing_latency default view for gauge. +rocketmq_consumer_queueing_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 +# TYPE rocketmq_consumer_connections gauge +# HELP rocketmq_consumer_connections default view for gauge. +rocketmq_consumer_connections{otel_scope_name="broker-meter",cluster="DefaultCluster",consume_mode="push",consumer_group="MQ_INST_test%GID_222",is_system="false",language="java",node_id="broker-a",node_type="broker",protocol_type="remoting",version="v4_8_0"} 1.0 1696905287361 +# TYPE rocketmq_rpc_latency histogram +# HELP rocketmq_rpc_latency Rpc latency +rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success"} 24.0 1696905287361 +rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success"} 128.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="1.0"} 17.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="3.0"} 18.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="5.0"} 19.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="7.0"} 19.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="10.0"} 20.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="100.0"} 24.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="1000.0"} 24.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="2000.0"} 24.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="3000.0"} 24.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="+Inf"} 24.0 1696905287361 +rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success"} 8.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="1.0"} 3.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="3.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="5.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="7.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="10.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="100.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="1000.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="2000.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="3000.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="+Inf"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success"} 54.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="1.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="3.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="5.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="7.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="10.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="100.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="1000.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="2000.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="3000.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="+Inf"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success"} 60.0 1696905287361 +rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success"} 108.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="1.0"} 54.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="3.0"} 54.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="5.0"} 54.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="7.0"} 54.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="10.0"} 55.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="100.0"} 60.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="1000.0"} 60.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="2000.0"} 60.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="3000.0"} 60.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="+Inf"} 60.0 1696905287361 +rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success"} 4.0 1696905287361 +rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success"} 210.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="1.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="3.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="5.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="7.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="10.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="100.0"} 4.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="1000.0"} 4.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="2000.0"} 4.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="3000.0"} 4.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="+Inf"} 4.0 1696905287361 +rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success"} 155.0 1696905287361 +rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success"} 2811756.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="1.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="3.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="5.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="7.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="10.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="100.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="1000.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="2000.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="3000.0"} 0.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="+Inf"} 155.0 1696905287361 +rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success"} 225.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="1.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="3.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="5.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="7.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="10.0"} 4.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="100.0"} 4.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="1000.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="2000.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="3000.0"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="+Inf"} 5.0 1696905287361 +rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway"} 565.0 1696905287361 +rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway"} 84.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="1.0"} 551.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="3.0"} 557.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="5.0"} 558.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="7.0"} 560.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="10.0"} 565.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="100.0"} 565.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="1000.0"} 565.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="2000.0"} 565.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="3000.0"} 565.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="+Inf"} 565.0 1696905287361 +rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success"} 6.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="1.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="3.0"} 1.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="5.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="7.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="10.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="100.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="1000.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="2000.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="3000.0"} 2.0 1696905287361 +rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="+Inf"} 2.0 1696905287361 +# TYPE rocketmq_storage_dispatch_behind_bytes gauge +# HELP rocketmq_storage_dispatch_behind_bytes default view for gauge. +rocketmq_storage_dispatch_behind_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 +# TYPE rocketmq_consumer_ready_messages gauge +# HELP rocketmq_consumer_ready_messages default view for gauge. +rocketmq_consumer_ready_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 +# TYPE rocketmq_consumer_lag_latency gauge +# HELP rocketmq_consumer_lag_latency default view for gauge. +rocketmq_consumer_lag_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 +# TYPE rocketmq_throughput_in_total counter +# HELP rocketmq_throughput_in_total default view for counter. +rocketmq_throughput_in_total{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361 +# TYPE rocketmq_pop_checkpoint_buffer_size gauge +# HELP rocketmq_pop_checkpoint_buffer_size default view for gauge. +rocketmq_pop_checkpoint_buffer_size{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 0.0 1696905287361 +# TYPE rocketmq_message_size histogram +# HELP rocketmq_message_size Incoming messages size +rocketmq_message_size_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361 +rocketmq_message_size_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361 +rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="1024.0"} 5.0 1696905287361 +rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="4096.0"} 5.0 1696905287361 +rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="524288.0"} 5.0 1696905287361 +rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="1048576.0"} 5.0 1696905287361 +rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="2097152.0"} 5.0 1696905287361 +rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="4194304.0"} 5.0 1696905287361 +rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="+Inf"} 5.0 1696905287361 +# TYPE rocketmq_consumer_inflight_messages gauge +# HELP rocketmq_consumer_inflight_messages default view for gauge. +rocketmq_consumer_inflight_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 +# TYPE rocketmq_producer_connections gauge +# HELP rocketmq_producer_connections default view for gauge. +rocketmq_producer_connections{otel_scope_name="broker-meter",cluster="DefaultCluster",language="java",node_id="broker-a",node_type="broker",protocol_type="remoting",version="v4_8_0"} 1.0 1696905287361 diff --git a/patch009-modify-CRLF-to-LF.patch b/patch009-modify-CRLF-to-LF.patch new file mode 100644 index 000000000..c9f1f1843 --- /dev/null +++ b/patch009-modify-CRLF-to-LF.patch @@ -0,0 +1,127 @@ +From 81cd734ee33586544694675bd1081f2c81024d22 Mon Sep 17 00:00:00 2001 +From: zhiliatom +Date: Thu, 28 Sep 2023 09:58:13 +0800 +Subject: [PATCH] modify line separater from CRLF to LF + +--- + .../rocketmq/common/constant/LoggerName.java | 110 +++++++++--------- + 1 file changed, 55 insertions(+), 55 deletions(-) + +diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +index cb04b00b3..5c7afab9f 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +@@ -1,55 +1,55 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package org.apache.rocketmq.common.constant; +- +-public class LoggerName { +- public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv"; +- public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; +- public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; +- public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; +- public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; +- public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; +- public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; +- public static final String CLIENT_LOGGER_NAME = "RocketmqClient"; +- public static final String ROCKETMQ_TRAFFIC_NAME = "RocketmqTraffic"; +- public static final String ROCKETMQ_REMOTING_NAME = "RocketmqRemoting"; +- public static final String TOOLS_LOGGER_NAME = "RocketmqTools"; +- public static final String COMMON_LOGGER_NAME = "RocketmqCommon"; +- public static final String STORE_LOGGER_NAME = "RocketmqStore"; +- public static final String STORE_ERROR_LOGGER_NAME = "RocketmqStoreError"; +- public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction"; +- public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock"; +- public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats"; +- public static final String DLQ_STATS_LOGGER_NAME = "RocketmqDLQStats"; +- public static final String DLQ_LOGGER_NAME = "RocketmqDLQ"; +- public static final String CONSUMER_STATS_LOGGER_NAME = "RocketmqConsumerStats"; +- public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial"; +- public static final String ACCOUNT_LOGGER_NAME = "RocketmqAccount"; +- public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl"; +- public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize"; +- public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication"; +- public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection"; +- public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark"; +- public static final String FILTER_LOGGER_NAME = "RocketmqFilter"; +- public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop"; +- public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover"; +- public static final String STDOUT_LOGGER_NAME = "STDOUT"; +- public static final String PROXY_LOGGER_NAME = "RocketmqProxy"; +- public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark"; +- public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = "RocketmqColdCtr"; +- public static final String ROCKSDB_LOGGER_NAME = "RocketmqRocksDB"; +-} ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.common.constant; ++ ++public class LoggerName { ++ public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv"; ++ public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; ++ public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; ++ public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; ++ public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; ++ public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; ++ public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; ++ public static final String CLIENT_LOGGER_NAME = "RocketmqClient"; ++ public static final String ROCKETMQ_TRAFFIC_NAME = "RocketmqTraffic"; ++ public static final String ROCKETMQ_REMOTING_NAME = "RocketmqRemoting"; ++ public static final String TOOLS_LOGGER_NAME = "RocketmqTools"; ++ public static final String COMMON_LOGGER_NAME = "RocketmqCommon"; ++ public static final String STORE_LOGGER_NAME = "RocketmqStore"; ++ public static final String STORE_ERROR_LOGGER_NAME = "RocketmqStoreError"; ++ public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction"; ++ public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock"; ++ public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats"; ++ public static final String DLQ_STATS_LOGGER_NAME = "RocketmqDLQStats"; ++ public static final String DLQ_LOGGER_NAME = "RocketmqDLQ"; ++ public static final String CONSUMER_STATS_LOGGER_NAME = "RocketmqConsumerStats"; ++ public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial"; ++ public static final String ACCOUNT_LOGGER_NAME = "RocketmqAccount"; ++ public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl"; ++ public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize"; ++ public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication"; ++ public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection"; ++ public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark"; ++ public static final String FILTER_LOGGER_NAME = "RocketmqFilter"; ++ public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop"; ++ public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover"; ++ public static final String STDOUT_LOGGER_NAME = "STDOUT"; ++ public static final String PROXY_LOGGER_NAME = "RocketmqProxy"; ++ public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark"; ++ public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = "RocketmqColdCtr"; ++ public static final String ROCKSDB_LOGGER_NAME = "RocketmqRocksDB"; ++} +-- +2.32.0.windows.2 + diff --git a/patch010-backport-add-some-fixes b/patch010-backport-add-some-fixes new file mode 100644 index 000000000..36d5e4a1f --- /dev/null +++ b/patch010-backport-add-some-fixes @@ -0,0 +1,1286 @@ +From b2deef179dbc6a9eb1a2b6dd7b652d95cb768295 Mon Sep 17 00:00:00 2001 +From: lizhimins <707364882@qq.com> +Date: Thu, 10 Aug 2023 10:38:47 +0800 +Subject: [PATCH 01/12] [ISSUE #7144] Accelerate the recovery speed of the + tiered storage module (#7145) + +--- + .../tieredstore/TieredDispatcher.java | 3 + + .../tieredstore/TieredMessageStore.java | 2 +- + .../common/TieredStoreExecutor.java | 25 ++-- + .../tieredstore/file/CompositeFlatFile.java | 15 +- + .../file/CompositeQueueFlatFile.java | 20 ++- + .../tieredstore/file/TieredCommitLog.java | 24 +++- + .../tieredstore/file/TieredFlatFile.java | 42 +++--- + .../file/TieredFlatFileManager.java | 135 ++++++++++-------- + .../metadata/FileSegmentMetadata.java | 26 +++- + .../tieredstore/TieredDispatcherTest.java | 15 +- + .../tieredstore/TieredMessageFetcherTest.java | 2 +- + .../file/CompositeQueueFlatFileTest.java | 2 +- + .../file/TieredFlatFileManagerTest.java | 7 +- + 13 files changed, 194 insertions(+), 124 deletions(-) + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +index bb58ea7dd..1746190cd 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +@@ -279,6 +279,9 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch + long upperBound = Math.min(dispatchOffset + maxCount, maxOffsetInQueue); + ConsumeQueue consumeQueue = (ConsumeQueue) defaultStore.getConsumeQueue(topic, queueId); + ++ logger.debug("DispatchFlatFile race, topic={}, queueId={}, cq range={}-{}, dispatch offset={}-{}", ++ topic, queueId, minOffsetInQueue, maxOffsetInQueue, dispatchOffset, upperBound - 1); ++ + for (; dispatchOffset < upperBound; dispatchOffset++) { + // get consume queue + SelectMappedBufferResult cqItem = consumeQueue.getIndexBuffer(dispatchOffset); +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +index 1f12410f2..ced1fb818 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +@@ -147,7 +147,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { + + if (!viaTieredStorage(topic, queueId, offset, maxMsgNums)) { +- logger.debug("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); ++ logger.trace("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); + return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); + } + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java +index 6eb3478b3..6dd0e8846 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java +@@ -43,18 +43,9 @@ public class TieredStoreExecutor { + public static ExecutorService compactIndexFileExecutor; + + public static void init() { +- dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); +- dispatchExecutor = new ThreadPoolExecutor( +- Math.max(2, Runtime.getRuntime().availableProcessors()), +- Math.max(16, Runtime.getRuntime().availableProcessors() * 4), +- 1000 * 60, +- TimeUnit.MILLISECONDS, +- dispatchThreadPoolQueue, +- new ThreadFactoryImpl("TieredCommonExecutor_")); +- + commonScheduledExecutor = new ScheduledThreadPoolExecutor( + Math.max(4, Runtime.getRuntime().availableProcessors()), +- new ThreadFactoryImpl("TieredCommonScheduledExecutor_")); ++ new ThreadFactoryImpl("TieredCommonExecutor_")); + + commitExecutor = new ScheduledThreadPoolExecutor( + Math.max(16, Runtime.getRuntime().availableProcessors() * 4), +@@ -62,7 +53,17 @@ public class TieredStoreExecutor { + + cleanExpiredFileExecutor = new ScheduledThreadPoolExecutor( + Math.max(4, Runtime.getRuntime().availableProcessors()), +- new ThreadFactoryImpl("TieredCleanExpiredFileExecutor_")); ++ new ThreadFactoryImpl("TieredCleanFileExecutor_")); ++ ++ dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++ dispatchExecutor = new ThreadPoolExecutor( ++ Math.max(2, Runtime.getRuntime().availableProcessors()), ++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++ 1000 * 60, ++ TimeUnit.MILLISECONDS, ++ dispatchThreadPoolQueue, ++ new ThreadFactoryImpl("TieredDispatchExecutor_"), ++ new ThreadPoolExecutor.DiscardOldestPolicy()); + + fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); + fetchDataExecutor = new ThreadPoolExecutor( +@@ -71,7 +72,7 @@ public class TieredStoreExecutor { + 1000 * 60, + TimeUnit.MILLISECONDS, + fetchDataThreadPoolQueue, +- new ThreadFactoryImpl("TieredFetchDataExecutor_")); ++ new ThreadFactoryImpl("TieredFetchExecutor_")); + + compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); + compactIndexFileExecutor = new ThreadPoolExecutor( +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java +index df4baf33f..5ad3a6ff3 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java +@@ -76,20 +76,15 @@ public class CompositeFlatFile implements CompositeAccess { + this.storeConfig = fileQueueFactory.getStoreConfig(); + this.readAheadFactor = this.storeConfig.getReadAheadMinFactor(); + this.metadataStore = TieredStoreUtil.getMetadataStore(this.storeConfig); +- this.dispatchOffset = new AtomicLong(); + this.compositeFlatFileLock = new ReentrantLock(); + this.inFlightRequestMap = new ConcurrentHashMap<>(); + this.commitLog = new TieredCommitLog(fileQueueFactory, filePath); + this.consumeQueue = new TieredConsumeQueue(fileQueueFactory, filePath); ++ this.dispatchOffset = new AtomicLong( ++ this.consumeQueue.isInitialized() ? this.getConsumeQueueCommitOffset() : -1L); + this.groupOffsetCache = this.initOffsetCache(); + } + +- protected void recoverMetadata() { +- if (!consumeQueue.isInitialized() && this.dispatchOffset.get() != -1) { +- consumeQueue.setBaseOffset(this.dispatchOffset.get() * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE); +- } +- } +- + private Cache initOffsetCache() { + return Caffeine.newBuilder() + .expireAfterWrite(2, TimeUnit.MINUTES) +@@ -310,10 +305,12 @@ public class CompositeFlatFile implements CompositeAccess { + + @Override + public void initOffset(long offset) { +- if (!consumeQueue.isInitialized()) { ++ if (consumeQueue.isInitialized()) { ++ dispatchOffset.set(this.getConsumeQueueCommitOffset()); ++ } else { + consumeQueue.setBaseOffset(offset * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE); ++ dispatchOffset.set(offset); + } +- dispatchOffset.set(offset); + } + + @Override +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java +index f6c0afed0..0a797f465 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java +@@ -36,8 +36,7 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { + public CompositeQueueFlatFile(TieredFileAllocator fileQueueFactory, MessageQueue messageQueue) { + super(fileQueueFactory, TieredStoreUtil.toPath(messageQueue)); + this.messageQueue = messageQueue; +- this.recoverTopicMetadata(); +- super.recoverMetadata(); ++ this.recoverQueueMetadata(); + this.indexFile = TieredFlatFileManager.getIndexFile(storeConfig); + } + +@@ -46,11 +45,12 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { + if (!consumeQueue.isInitialized()) { + queueMetadata.setMinOffset(offset); + queueMetadata.setMaxOffset(offset); ++ metadataStore.updateQueue(queueMetadata); + } + super.initOffset(offset); + } + +- public void recoverTopicMetadata() { ++ public void recoverQueueMetadata() { + TopicMetadata topicMetadata = this.metadataStore.getTopic(messageQueue.getTopic()); + if (topicMetadata == null) { + topicMetadata = this.metadataStore.addTopic(messageQueue.getTopic(), -1L); +@@ -64,18 +64,16 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { + if (queueMetadata.getMaxOffset() < queueMetadata.getMinOffset()) { + queueMetadata.setMaxOffset(queueMetadata.getMinOffset()); + } +- this.dispatchOffset.set(queueMetadata.getMaxOffset()); + } + +- public void persistMetadata() { ++ public void flushMetadata() { + try { +- if (consumeQueue.getCommitOffset() < queueMetadata.getMinOffset()) { +- return; +- } +- queueMetadata.setMaxOffset(consumeQueue.getCommitOffset() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE); ++ queueMetadata.setMinOffset(super.getConsumeQueueMinOffset()); ++ queueMetadata.setMaxOffset(super.getConsumeQueueMaxOffset()); + metadataStore.updateQueue(queueMetadata); + } catch (Exception e) { +- LOGGER.error("CompositeFlatFile#flushMetadata: update queue metadata failed: topic: {}, queue: {}", messageQueue.getTopic(), messageQueue.getQueueId(), e); ++ LOGGER.error("CompositeFlatFile#flushMetadata error, topic: {}, queue: {}", ++ messageQueue.getTopic(), messageQueue.getQueueId(), e); + } + } + +@@ -114,7 +112,7 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { + @Override + public void shutdown() { + super.shutdown(); +- metadataStore.updateQueue(queueMetadata); ++ this.flushMetadata(); + } + + @Override +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java +index 80e1bce50..0e5f79132 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java +@@ -50,7 +50,7 @@ public class TieredCommitLog { + this.storeConfig = fileQueueFactory.getStoreConfig(); + this.flatFile = fileQueueFactory.createFlatFileForCommitLog(filePath); + this.minConsumeQueueOffset = new AtomicLong(NOT_EXIST_MIN_OFFSET); +- this.correctMinOffset(); ++ this.correctMinOffsetAsync(); + } + + @VisibleForTesting +@@ -91,17 +91,26 @@ public class TieredCommitLog { + return flatFile.getFileToWrite().getMaxTimestamp(); + } + +- public synchronized long correctMinOffset() { ++ public long correctMinOffset() { ++ try { ++ return correctMinOffsetAsync().get(); ++ } catch (Exception e) { ++ log.error("Correct min offset failed in clean expired file", e); ++ } ++ return NOT_EXIST_MIN_OFFSET; ++ } ++ ++ public synchronized CompletableFuture correctMinOffsetAsync() { + if (flatFile.getFileSegmentCount() == 0) { + this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET); +- return NOT_EXIST_MIN_OFFSET; ++ return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET); + } + + // queue offset field length is 8 + int length = MessageBufferUtil.QUEUE_OFFSET_POSITION + 8; + if (flatFile.getCommitOffset() - flatFile.getMinOffset() < length) { + this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET); +- return NOT_EXIST_MIN_OFFSET; ++ return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET); + } + + try { +@@ -109,7 +118,8 @@ public class TieredCommitLog { + .thenApply(buffer -> { + long offset = MessageBufferUtil.getQueueOffset(buffer); + minConsumeQueueOffset.set(offset); +- log.info("Correct commitlog min cq offset success, filePath={}, min cq offset={}, range={}-{}", ++ log.debug("Correct commitlog min cq offset success, " + ++ "filePath={}, min cq offset={}, commitlog range={}-{}", + flatFile.getFilePath(), offset, flatFile.getMinOffset(), flatFile.getCommitOffset()); + return offset; + }) +@@ -117,11 +127,11 @@ public class TieredCommitLog { + log.warn("Correct commitlog min cq offset error, filePath={}, range={}-{}", + flatFile.getFilePath(), flatFile.getMinOffset(), flatFile.getCommitOffset(), throwable); + return minConsumeQueueOffset.get(); +- }).get(); ++ }); + } catch (Exception e) { + log.error("Correct commitlog min cq offset error, filePath={}", flatFile.getFilePath(), e); + } +- return minConsumeQueueOffset.get(); ++ return CompletableFuture.completedFuture(minConsumeQueueOffset.get()); + } + + public AppendResult append(ByteBuffer byteBuf) { +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +index 75ce8d89f..426c4e09d 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +@@ -16,6 +16,7 @@ + */ + package org.apache.rocketmq.tieredstore.file; + ++import com.alibaba.fastjson.JSON; + import com.google.common.annotations.VisibleForTesting; + import java.nio.ByteBuffer; + import java.util.ArrayList; +@@ -24,6 +25,7 @@ import java.util.Comparator; + import java.util.HashSet; + import java.util.LinkedList; + import java.util.List; ++import java.util.Objects; + import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.CopyOnWriteArrayList; +@@ -178,32 +180,26 @@ public class TieredFlatFile { + private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fileSegment) { + + FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( +- fileSegment.getPath(), fileSegment.getFileType(), fileSegment.getBaseOffset()); +- +- if (metadata != null) { +- return metadata; +- } ++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); + + // Note: file segment path may not the same as file base path, use base path here. +- metadata = new FileSegmentMetadata( +- this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); +- +- if (fileSegment.isClosed()) { +- metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); ++ if (metadata == null) { ++ metadata = new FileSegmentMetadata( ++ this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); ++ metadata.setCreateTimestamp(fileSegment.getMinTimestamp()); ++ metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); ++ metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++ if (fileSegment.isClosed()) { ++ metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); ++ } ++ this.tieredMetadataStore.updateFileSegment(metadata); + } +- +- metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); +- metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); +- +- // Submit to persist +- this.tieredMetadataStore.updateFileSegment(metadata); + return metadata; + } + + /** + * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full + */ +- @VisibleForTesting + public void updateFileSegment(TieredFileSegment fileSegment) { + FileSegmentMetadata segmentMetadata = getOrCreateFileSegmentMetadata(fileSegment); + +@@ -219,9 +215,16 @@ public class TieredFlatFile { + } + + segmentMetadata.setSize(fileSegment.getCommitPosition()); +- segmentMetadata.setBeginTimestamp(fileSegment.getMinTimestamp()); + segmentMetadata.setEndTimestamp(fileSegment.getMaxTimestamp()); +- this.tieredMetadataStore.updateFileSegment(segmentMetadata); ++ ++ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( ++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); ++ ++ if (!Objects.equals(metadata, segmentMetadata)) { ++ this.tieredMetadataStore.updateFileSegment(segmentMetadata); ++ logger.debug("TieredFlatFile#UpdateSegmentMetadata, filePath: {}, content: {}", ++ segmentMetadata.getPath(), JSON.toJSONString(segmentMetadata)); ++ } + } + + private void checkAndFixFileSize() { +@@ -257,6 +260,7 @@ public class TieredFlatFile { + logger.warn("TieredFlatFile#checkAndFixFileSize: fix last file {} size: origin: {}, actual: {}", + lastFile.getPath(), lastFile.getCommitOffset() - lastFile.getBaseOffset(), lastFileSize); + lastFile.initPosition(lastFileSize); ++ this.updateFileSegment(lastFile); + } + } + } +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +index aeca44b8c..e9ae4a5a5 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +@@ -16,16 +16,19 @@ + */ + package org.apache.rocketmq.tieredstore.file; + ++import com.google.common.base.Stopwatch; + import com.google.common.collect.ImmutableList; + import java.util.ArrayList; + import java.util.List; + import java.util.Random; ++import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; +-import java.util.concurrent.Future; ++import java.util.concurrent.Semaphore; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicLong; + import javax.annotation.Nullable; ++import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +@@ -36,6 +39,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + + public class TieredFlatFileManager { + ++ private static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); + + private static volatile TieredFlatFileManager instance; +@@ -44,7 +48,7 @@ public class TieredFlatFileManager { + private final TieredMetadataStore metadataStore; + private final TieredMessageStoreConfig storeConfig; + private final TieredFileAllocator tieredFileAllocator; +- private final ConcurrentMap queueFlatFileMap; ++ private final ConcurrentMap flatFileConcurrentMap; + + public TieredFlatFileManager(TieredMessageStoreConfig storeConfig) + throws ClassNotFoundException, NoSuchMethodException { +@@ -52,23 +56,20 @@ public class TieredFlatFileManager { + this.storeConfig = storeConfig; + this.metadataStore = TieredStoreUtil.getMetadataStore(storeConfig); + this.tieredFileAllocator = new TieredFileAllocator(storeConfig); +- this.queueFlatFileMap = new ConcurrentHashMap<>(); ++ this.flatFileConcurrentMap = new ConcurrentHashMap<>(); + this.doScheduleTask(); + } + + public static TieredFlatFileManager getInstance(TieredMessageStoreConfig storeConfig) { +- if (storeConfig == null) { ++ if (storeConfig == null || instance != null) { + return instance; + } +- +- if (instance == null) { +- synchronized (TieredFlatFileManager.class) { +- if (instance == null) { +- try { +- instance = new TieredFlatFileManager(storeConfig); +- } catch (Exception e) { +- logger.error("TieredFlatFileManager#getInstance: create flat file manager failed", e); +- } ++ synchronized (TieredFlatFileManager.class) { ++ if (instance == null) { ++ try { ++ instance = new TieredFlatFileManager(storeConfig); ++ } catch (Exception e) { ++ logger.error("Construct FlatFileManager instance error", e); + } + } + } +@@ -88,7 +89,7 @@ public class TieredFlatFileManager { + TieredStoreUtil.RMQ_SYS_TIERED_STORE_INDEX_TOPIC, storeConfig.getBrokerName(), 0)); + indexFile = new TieredIndexFile(new TieredFileAllocator(storeConfig), filePath); + } catch (Exception e) { +- logger.error("TieredFlatFileManager#getIndexFile: create index file failed", e); ++ logger.error("Construct FlatFileManager indexFile error", e); + } + } + } +@@ -105,7 +106,7 @@ public class TieredFlatFileManager { + flatFile.commitCommitLog(); + } catch (Throwable e) { + MessageQueue mq = flatFile.getMessageQueue(); +- logger.error("commit commitLog periodically failed: topic: {}, queue: {}", ++ logger.error("Commit commitLog periodically failed: topic: {}, queue: {}", + mq.getTopic(), mq.getQueueId(), e); + } + }, delay, TimeUnit.MILLISECONDS); +@@ -114,7 +115,7 @@ public class TieredFlatFileManager { + flatFile.commitConsumeQueue(); + } catch (Throwable e) { + MessageQueue mq = flatFile.getMessageQueue(); +- logger.error("commit consumeQueue periodically failed: topic: {}, queue: {}", ++ logger.error("Commit consumeQueue periodically failed: topic: {}, queue: {}", + mq.getTopic(), mq.getQueueId(), e); + } + }, delay, TimeUnit.MILLISECONDS); +@@ -125,7 +126,7 @@ public class TieredFlatFileManager { + indexFile.commit(true); + } + } catch (Throwable e) { +- logger.error("commit indexFile periodically failed", e); ++ logger.error("Commit indexFile periodically failed", e); + } + }, 0, TimeUnit.MILLISECONDS); + } +@@ -160,7 +161,7 @@ public class TieredFlatFileManager { + try { + doCommit(); + } catch (Throwable e) { +- logger.error("commit flat file periodically failed: ", e); ++ logger.error("Commit flat file periodically failed: ", e); + } + }, 60, 60, TimeUnit.SECONDS); + +@@ -168,49 +169,73 @@ public class TieredFlatFileManager { + try { + doCleanExpiredFile(); + } catch (Throwable e) { +- logger.error("clean expired flat file failed: ", e); ++ logger.error("Clean expired flat file failed: ", e); + } + }, 30, 30, TimeUnit.SECONDS); + } + + public boolean load() { ++ Stopwatch stopwatch = Stopwatch.createStarted(); + try { +- AtomicLong topicSequenceNumber = new AtomicLong(); +- List> futureList = new ArrayList<>(); +- queueFlatFileMap.clear(); +- metadataStore.iterateTopic(topicMetadata -> { ++ flatFileConcurrentMap.clear(); ++ this.recoverSequenceNumber(); ++ this.recoverTieredFlatFile(); ++ logger.info("Message store recover end, total cost={}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); ++ } catch (Exception e) { ++ long costTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); ++ logger.info("Message store recover error, total cost={}ms", costTime); ++ BROKER_LOG.error("Message store recover error, total cost={}ms", costTime, e); ++ return false; ++ } ++ return true; ++ } ++ ++ public void recoverSequenceNumber() { ++ AtomicLong topicSequenceNumber = new AtomicLong(); ++ metadataStore.iterateTopic(topicMetadata -> { ++ if (topicMetadata != null && topicMetadata.getTopicId() > 0) { + topicSequenceNumber.set(Math.max(topicSequenceNumber.get(), topicMetadata.getTopicId())); +- Future future = TieredStoreExecutor.dispatchExecutor.submit(() -> { +- if (topicMetadata.getStatus() != 0) { +- return; +- } ++ } ++ }); ++ metadataStore.setTopicSequenceNumber(topicSequenceNumber.incrementAndGet()); ++ } ++ ++ public void recoverTieredFlatFile() { ++ Semaphore semaphore = new Semaphore((int) (TieredStoreExecutor.QUEUE_CAPACITY * 0.75)); ++ List> futures = new ArrayList<>(); ++ metadataStore.iterateTopic(topicMetadata -> { ++ try { ++ semaphore.acquire(); ++ CompletableFuture future = CompletableFuture.runAsync(() -> { + try { +- metadataStore.iterateQueue(topicMetadata.getTopic(), +- queueMetadata -> getOrCreateFlatFileIfAbsent( +- new MessageQueue(topicMetadata.getTopic(), +- storeConfig.getBrokerName(), +- queueMetadata.getQueue().getQueueId()))); ++ Stopwatch subWatch = Stopwatch.createStarted(); ++ if (topicMetadata.getStatus() != 0) { ++ return; ++ } ++ AtomicLong queueCount = new AtomicLong(); ++ metadataStore.iterateQueue(topicMetadata.getTopic(), queueMetadata -> { ++ this.getOrCreateFlatFileIfAbsent(new MessageQueue(topicMetadata.getTopic(), ++ storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId())); ++ queueCount.incrementAndGet(); ++ }); ++ logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", ++ topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); + } catch (Exception e) { +- logger.error("load mq composite flat file from metadata failed", e); ++ logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e); ++ } finally { ++ semaphore.release(); + } +- }); +- futureList.add(future); +- }); +- +- // Wait for load all metadata done +- for (Future future : futureList) { +- future.get(); ++ }, TieredStoreExecutor.commitExecutor); ++ futures.add(future); ++ } catch (Exception e) { ++ throw new RuntimeException(e); + } +- metadataStore.setTopicSequenceNumber(topicSequenceNumber.incrementAndGet()); +- } catch (Exception e) { +- logger.error("load mq composite flat file from metadata failed", e); +- return false; +- } +- return true; ++ }); ++ CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + } + + public void cleanup() { +- queueFlatFileMap.clear(); ++ flatFileConcurrentMap.clear(); + cleanStaticReference(); + } + +@@ -221,27 +246,25 @@ public class TieredFlatFileManager { + + @Nullable + public CompositeQueueFlatFile getOrCreateFlatFileIfAbsent(MessageQueue messageQueue) { +- return queueFlatFileMap.computeIfAbsent(messageQueue, mq -> { ++ return flatFileConcurrentMap.computeIfAbsent(messageQueue, mq -> { + try { +- logger.debug("TieredFlatFileManager#getOrCreateFlatFileIfAbsent: " + +- "try to create new flat file: topic: {}, queueId: {}", ++ logger.debug("Create new TopicFlatFile, topic: {}, queueId: {}", + messageQueue.getTopic(), messageQueue.getQueueId()); + return new CompositeQueueFlatFile(tieredFileAllocator, mq); + } catch (Exception e) { +- logger.error("TieredFlatFileManager#getOrCreateFlatFileIfAbsent: " + +- "create new flat file: topic: {}, queueId: {}", ++ logger.debug("Create new TopicFlatFile failed, topic: {}, queueId: {}", + messageQueue.getTopic(), messageQueue.getQueueId(), e); +- return null; + } ++ return null; + }); + } + + public CompositeQueueFlatFile getFlatFile(MessageQueue messageQueue) { +- return queueFlatFileMap.get(messageQueue); ++ return flatFileConcurrentMap.get(messageQueue); + } + + public ImmutableList deepCopyFlatFileToList() { +- return ImmutableList.copyOf(queueFlatFileMap.values()); ++ return ImmutableList.copyOf(flatFileConcurrentMap.values()); + } + + public void shutdown() { +@@ -270,7 +293,7 @@ public class TieredFlatFileManager { + } + + // delete memory reference +- CompositeQueueFlatFile flatFile = queueFlatFileMap.remove(mq); ++ CompositeQueueFlatFile flatFile = flatFileConcurrentMap.remove(mq); + if (flatFile != null) { + MessageQueue messageQueue = flatFile.getMessageQueue(); + logger.info("TieredFlatFileManager#destroyCompositeFile: " + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java +index 1b232fc75..2f0fd71de 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java +@@ -16,6 +16,8 @@ + */ + package org.apache.rocketmq.tieredstore.metadata; + ++import java.util.Objects; ++ + public class FileSegmentMetadata { + + public static final int STATUS_NEW = 0; +@@ -43,7 +45,6 @@ public class FileSegmentMetadata { + this.baseOffset = baseOffset; + this.type = type; + this.status = STATUS_NEW; +- this.createTimestamp = System.currentTimeMillis(); + } + + public void markSealed() { +@@ -122,4 +123,27 @@ public class FileSegmentMetadata { + public void setSealTimestamp(long sealTimestamp) { + this.sealTimestamp = sealTimestamp; + } ++ ++ @Override ++ public boolean equals(Object o) { ++ if (this == o) ++ return true; ++ if (o == null || getClass() != o.getClass()) ++ return false; ++ FileSegmentMetadata metadata = (FileSegmentMetadata) o; ++ return size == metadata.size ++ && baseOffset == metadata.baseOffset ++ && status == metadata.status ++ && path.equals(metadata.path) ++ && type == metadata.type ++ && createTimestamp == metadata.createTimestamp ++ && beginTimestamp == metadata.beginTimestamp ++ && endTimestamp == metadata.endTimestamp ++ && sealTimestamp == metadata.sealTimestamp; ++ } ++ ++ @Override ++ public int hashCode() { ++ return Objects.hash(type, path, baseOffset, status, size, createTimestamp, beginTimestamp, endTimestamp, sealTimestamp); ++ } + } +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java +index e6adef1d1..5791dc9a4 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java +@@ -116,19 +116,20 @@ public class TieredDispatcherTest { + buffer3.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 9); + flatFile.appendCommitLog(buffer3); + flatFile.commitCommitLog(); +- Assert.assertEquals(10, flatFile.getDispatchOffset()); ++ Assert.assertEquals(9 + 1, flatFile.getDispatchOffset()); ++ Assert.assertEquals(9, flatFile.getCommitLogDispatchCommitOffset()); + + dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer1); + dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer2); + dispatcher.buildConsumeQueueAndIndexFile(); + Assert.assertEquals(7, flatFile.getConsumeQueueMaxOffset()); +- Assert.assertEquals(7, flatFile.getDispatchOffset()); + + dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 7, 7, 0, 0, buffer1); + dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer2); + dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer3); + dispatcher.buildConsumeQueueAndIndexFile(); +- Assert.assertEquals(10, flatFile.getConsumeQueueMaxOffset()); ++ Assert.assertEquals(6, flatFile.getConsumeQueueMinOffset()); ++ Assert.assertEquals(9 + 1, flatFile.getConsumeQueueMaxOffset()); + } + + @Test +@@ -142,6 +143,7 @@ public class TieredDispatcherTest { + Mockito.when(defaultStore.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(0L); + Mockito.when(defaultStore.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(9L); + ++ // mock cq item, position = 7 + ByteBuffer cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE); + cqItem.putLong(7); + cqItem.putInt(MessageBufferUtilTest.MSG_LEN); +@@ -150,13 +152,13 @@ public class TieredDispatcherTest { + SelectMappedBufferResult mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null); + Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(6)).thenReturn(mockResult); + ++ // mock cq item, position = 8 + cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE); + cqItem.putLong(8); + cqItem.putInt(MessageBufferUtilTest.MSG_LEN); + cqItem.putLong(1); + cqItem.flip(); + mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null); +- + Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(7)).thenReturn(mockResult); + + mockResult = new SelectMappedBufferResult(0, MessageBufferUtilTest.buildMockedMessageBuffer(), MessageBufferUtilTest.MSG_LEN, null); +@@ -167,7 +169,10 @@ public class TieredDispatcherTest { + mockResult = new SelectMappedBufferResult(0, msg, MessageBufferUtilTest.MSG_LEN, null); + Mockito.when(defaultStore.selectOneMessageByOffset(8, MessageBufferUtilTest.MSG_LEN)).thenReturn(mockResult); + +- dispatcher.dispatchFlatFile(flatFileManager.getOrCreateFlatFileIfAbsent(mq)); ++ CompositeQueueFlatFile flatFile = flatFileManager.getOrCreateFlatFileIfAbsent(mq); ++ Assert.assertNotNull(flatFile); ++ flatFile.initOffset(7); ++ dispatcher.dispatchFlatFile(flatFile); + Assert.assertEquals(8, flatFileManager.getFlatFile(mq).getDispatchOffset()); + } + } +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java +index d75b2f916..774c6cf64 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java +@@ -23,6 +23,7 @@ import java.util.Objects; + import java.util.concurrent.TimeUnit; + import org.apache.commons.lang3.SystemUtils; + import org.apache.commons.lang3.tuple.Triple; ++import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.store.DispatchRequest; + import org.apache.rocketmq.store.GetMessageResult; +@@ -40,7 +41,6 @@ import org.apache.rocketmq.tieredstore.file.TieredIndexFile; + import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; + import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest; + import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; +-import org.apache.rocketmq.common.BoundaryType; + import org.awaitility.Awaitility; + import org.junit.After; + import org.junit.Assert; +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java +index 27efe111e..2e028ada3 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java +@@ -119,7 +119,7 @@ public class CompositeQueueFlatFileTest { + Assert.assertEquals(AppendResult.SUCCESS, result); + + file.commit(true); +- file.persistMetadata(); ++ file.flushMetadata(); + + QueueMetadata queueMetadata = metadataStore.getQueue(mq); + Assert.assertEquals(53, queueMetadata.getMaxOffset()); +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java +index b7528c5e4..20fe4dd70 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java +@@ -72,10 +72,15 @@ public class TieredFlatFileManagerTest { + + CompositeFlatFile flatFile = flatFileManager.getFlatFile(mq); + Assert.assertNotNull(flatFile); +- Assert.assertEquals(100, flatFile.getDispatchOffset()); ++ Assert.assertEquals(-1L, flatFile.getDispatchOffset()); ++ flatFile.initOffset(100L); ++ Assert.assertEquals(100L, flatFile.getDispatchOffset()); ++ flatFile.initOffset(200L); ++ Assert.assertEquals(100L, flatFile.getDispatchOffset()); + + CompositeFlatFile flatFile1 = flatFileManager.getFlatFile(mq1); + Assert.assertNotNull(flatFile1); ++ flatFile1.initOffset(200L); + Assert.assertEquals(200, flatFile1.getDispatchOffset()); + + flatFileManager.destroyCompositeFile(mq); +-- +2.32.0.windows.2 + + +From 99b39a35f29e491862296d56b7938a995d153974 Mon Sep 17 00:00:00 2001 +From: ShuangxiDing +Date: Thu, 10 Aug 2023 11:28:39 +0800 +Subject: [PATCH 02/12] [ISSUE #7115] Fix grpc response message NPE (#7116) + +--- + .../apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java +index 0b3c85ea6..efa879a9c 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java +@@ -92,7 +92,7 @@ public class ResponseBuilder { + public Status buildStatus(Code code, String message) { + return Status.newBuilder() + .setCode(code) +- .setMessage(message) ++ .setMessage(message != null ? message : code.name()) + .build(); + } + +-- +2.32.0.windows.2 + + +From c0ba453f38183266cf9a69be754e620311e1923b Mon Sep 17 00:00:00 2001 +From: caigy +Date: Thu, 10 Aug 2023 14:08:17 +0800 +Subject: [PATCH 03/12] [ISSUE #7129] Fix resource collisions in acl tests + (#7130) + +* run acl tests sequencially to avoid collision + +* disable reuseForks for acl like broker + +* Revert "[ISSUE #7135] Temporarily ignoring plainAccessValidator test (#7135)" + +This reverts commit 6bc2c8474a0ce1e2833c82dffea7b1d8f718fcd7. +--- + acl/pom.xml | 13 +++++++++++++ + .../acl/plain/PlainAccessControlFlowTest.java | 5 ----- + .../acl/plain/PlainAccessValidatorTest.java | 3 --- + .../acl/plain/PlainPermissionManagerTest.java | 3 --- + 4 files changed, 13 insertions(+), 11 deletions(-) + +diff --git a/acl/pom.xml b/acl/pom.xml +index 67bfcb8d2..989c0cf77 100644 +--- a/acl/pom.xml ++++ b/acl/pom.xml +@@ -74,4 +74,17 @@ + test + + ++ ++ ++ ++ ++ maven-surefire-plugin ++ ${maven-surefire-plugin.version} ++ ++ 1 ++ false ++ ++ ++ ++ + +diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java +index e7fd0932f..519345714 100644 +--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java ++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java +@@ -31,7 +31,6 @@ import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; + import org.junit.Assert; +-import org.junit.Ignore; + import org.junit.Test; + + import java.io.File; +@@ -44,7 +43,6 @@ import java.util.Collections; + import java.util.LinkedList; + import java.util.List; + +- + /** + *

In this class, we'll test the following scenarios, each containing several consecutive operations on ACL, + *

like updating and deleting ACL, changing config files and checking validations. +@@ -52,9 +50,6 @@ import java.util.List; + *

Case 2: Only conf/acl/plain_acl.yml exists; + *

Case 3: Both conf/plain_acl.yml and conf/acl/plain_acl.yml exists. + */ +- +-// Ignore this test case as it is currently unable to pass on ubuntu workflow +-@Ignore + public class PlainAccessControlFlowTest { + public static final String DEFAULT_TOPIC = "topic-acl"; + +diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java +index a3a925758..ef0cffbdc 100644 +--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java ++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java +@@ -56,11 +56,8 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; + import org.junit.After; + import org.junit.Assert; + import org.junit.Before; +-import org.junit.Ignore; + import org.junit.Test; + +-// Ignore this test case as it is currently unable to pass on ubuntu workflow +-@Ignore + public class PlainAccessValidatorTest { + + private PlainAccessValidator plainAccessValidator; +diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java +index aa7539f3a..941d8c779 100644 +--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java ++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java +@@ -29,7 +29,6 @@ import org.assertj.core.api.Assertions; + import org.assertj.core.util.Lists; + import org.junit.Assert; + import org.junit.Before; +-import org.junit.Ignore; + import org.junit.Test; + + import java.io.File; +@@ -42,8 +41,6 @@ import java.util.List; + import java.util.Map; + import java.util.Set; + +-// Ignore this test case as it is currently unable to pass on ubuntu workflow +-@Ignore + public class PlainPermissionManagerTest { + + PlainPermissionManager plainPermissionManager; +-- +2.32.0.windows.2 + + +From 8741ff8c9b3bdbfc97976285affa7ea35c81243c Mon Sep 17 00:00:00 2001 +From: ShuangxiDing +Date: Thu, 10 Aug 2023 17:41:15 +0800 +Subject: [PATCH 04/12] [ISSUE #7153] Add switch for MIXED message type (#7154) + +Add a switch for MIXED message type when creating a Topic in the Broker. +--- + .../broker/processor/AdminBrokerProcessor.java | 8 ++++++++ + .../java/org/apache/rocketmq/common/BrokerConfig.java | 10 ++++++++++ + 2 files changed, 18 insertions(+) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +index a6ce03dc2..bbddcec2d 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +@@ -59,6 +59,7 @@ import org.apache.rocketmq.common.TopicConfig; + import org.apache.rocketmq.common.UnlockCallback; + import org.apache.rocketmq.common.UtilAll; + import org.apache.rocketmq.common.attribute.AttributeParser; ++import org.apache.rocketmq.common.attribute.TopicMessageType; + import org.apache.rocketmq.common.constant.ConsumeInitMode; + import org.apache.rocketmq.common.constant.FIleReadaheadMode; + import org.apache.rocketmq.common.constant.LoggerName; +@@ -439,6 +440,13 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + String attributesModification = requestHeader.getAttributes(); + topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification)); + ++ if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED ++ && !brokerController.getBrokerConfig().isEnableMixedMessageType()) { ++ response.setCode(ResponseCode.SYSTEM_ERROR); ++ response.setRemark("MIXED message type is not supported."); ++ return response; ++ } ++ + try { + this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); + if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +index a815636b1..99a5db5ad 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +@@ -393,6 +393,8 @@ public class BrokerConfig extends BrokerIdentity { + */ + private boolean enableSingleTopicRegister = false; + ++ private boolean enableMixedMessageType = false; ++ + public long getMaxPopPollingSize() { + return maxPopPollingSize; + } +@@ -1712,4 +1714,12 @@ public class BrokerConfig extends BrokerIdentity { + public void setEnableSingleTopicRegister(boolean enableSingleTopicRegister) { + this.enableSingleTopicRegister = enableSingleTopicRegister; + } ++ ++ public boolean isEnableMixedMessageType() { ++ return enableMixedMessageType; ++ } ++ ++ public void setEnableMixedMessageType(boolean enableMixedMessageType) { ++ this.enableMixedMessageType = enableMixedMessageType; ++ } + } +-- +2.32.0.windows.2 + + +From f534501855f8edbcb58f5b856973bf1027b5cf3a Mon Sep 17 00:00:00 2001 +From: Steven +Date: Fri, 11 Aug 2023 10:25:48 +0800 +Subject: [PATCH 05/12] [Feature 7155] add errlog when cmd err (#7157) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Co-authored-by: 十真 +--- + .../src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java +index b00bad3c5..5a8a7cd54 100644 +--- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java ++++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java +@@ -52,6 +52,7 @@ public class ServerUtil { + System.exit(0); + } + } catch (ParseException e) { ++ System.err.println(e.getMessage()); + hf.printHelp(appName, options, true); + System.exit(1); + } +-- +2.32.0.windows.2 + + +From db58f00c0fe0f129611d654291f2177de55dc9ff Mon Sep 17 00:00:00 2001 +From: Zhouxiang Zhan +Date: Fri, 11 Aug 2023 19:18:30 +0800 +Subject: [PATCH 06/12] [ISSUE #7169] Change metadataThreadPoolQueueCapacity to + 100000 (#7170) + +--- + .../main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +index 4f57a7052..39caaa0d9 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +@@ -165,7 +165,7 @@ public class ProxyConfig implements ConfigFile { + private int subscriptionGroupConfigCacheExpiredInSeconds = 20; + private int subscriptionGroupConfigCacheMaxNum = 20000; + private int metadataThreadPoolNums = 3; +- private int metadataThreadPoolQueueCapacity = 1000; ++ private int metadataThreadPoolQueueCapacity = 100000; + + private int transactionHeartbeatThreadPoolNums = 20; + private int transactionHeartbeatThreadPoolQueueCapacity = 200; +-- +2.32.0.windows.2 + + +From 1f04e68a2e331ab035b791280c5a91b60fe0c85f Mon Sep 17 00:00:00 2001 +From: yx9o +Date: Sat, 12 Aug 2023 21:12:22 +0800 +Subject: [PATCH 07/12] [ISSUE #7172] Unified Chinese for Name Server (#7173) + +--- + docs/cn/concept.md | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/docs/cn/concept.md b/docs/cn/concept.md +index cb2c863bd..3d67e9371 100644 +--- a/docs/cn/concept.md ++++ b/docs/cn/concept.md +@@ -17,7 +17,7 @@ RocketMQ主要由 Producer、Broker、Consumer 三部分组成，其中Producer + 消息中转角色，负责存储消息、转发消息。代理服务器在RocketMQ系统中负责接收从生产者发送来的消息并存储、同时为消费者的拉取请求作准备。代理服务器也存储消息相关的元数据，包括消费者组、消费进度偏移和主题和队列消息等。 + + ## 6 名字服务（Name Server） +- 名称服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。 ++名字服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。 + + ## 7 拉取式消费（Pull Consumer） + Consumer消费的一种类型，应用通常主动调用Consumer的拉消息方法从Broker服务器拉消息、主动权由应用控制。一旦获取了批量消息，应用就会启动消费过程。 +-- +2.32.0.windows.2 + + +From 25005060bbace477eeaaf4c0142cece5213efbbf Mon Sep 17 00:00:00 2001 +From: yx9o +Date: Sun, 13 Aug 2023 20:52:17 +0800 +Subject: [PATCH 08/12] [ISSUE #7176] Correct mismatched logs (#7177) + +--- + .../org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +index 0055a1cc8..f7a95f0a6 100644 +--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java ++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +@@ -522,7 +522,7 @@ public class RouteInfoManager { + this.lock.writeLock().unlock(); + } + } catch (Exception e) { +- log.error("wipeWritePermOfBrokerByLock Exception", e); ++ log.error("addWritePermOfBrokerByLock Exception", e); + } + return 0; + } +-- +2.32.0.windows.2 + + +From ac411daa27117e9115a8fc5e2d5753085f009ed9 Mon Sep 17 00:00:00 2001 +From: yx9o +Date: Tue, 15 Aug 2023 08:31:00 +0800 +Subject: [PATCH 09/12] [ISSUE #7183] Correct mismatched commandDesc (#7184) + +--- + .../tools/command/topic/RemappingStaticTopicSubCommand.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java +index 849f680d0..2a08fdb5b 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java +@@ -47,7 +47,7 @@ public class RemappingStaticTopicSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Update or create static topic, which has fixed number of queues"; ++ return "Remapping static topic."; + } + + @Override +-- +2.32.0.windows.2 + + +From 55e0cdb2af3ab75a6d892f919d60797f17a99fda Mon Sep 17 00:00:00 2001 +From: redlsz +Date: Tue, 15 Aug 2023 19:19:45 +0800 +Subject: [PATCH 10/12] fix: IndexOutOfBoundsException when process pop + response (#7003) + +--- + .../org/apache/rocketmq/client/impl/MQClientAPIImpl.java | 5 ++++- + .../rocketmq/proxy/service/message/LocalMessageService.java | 5 ++++- + .../rocketmq/remoting/protocol/header/ExtraInfoUtil.java | 4 ++++ + 3 files changed, 12 insertions(+), 2 deletions(-) + +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +index 708a6acd1..5101ffc8e 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +@@ -1174,7 +1174,10 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET))); + continue; + } +- key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId()); ++ // Value of POP_CK is used to determine whether it is a pop retry, ++ // cause topic could be rewritten by broker. ++ key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), ++ messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId()); + if (!sortMap.containsKey(key)) { + sortMap.put(key, new ArrayList<>(4)); + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +index 115c140ff..eb2c4d9ee 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +@@ -249,7 +249,10 @@ public class LocalMessageService implements MessageService { + // + Map> sortMap = new HashMap<>(16); + for (MessageExt messageExt : messageExtList) { +- String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId()); ++ // Value of POP_CK is used to determine whether it is a pop retry, ++ // cause topic could be rewritten by broker. ++ String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), ++ messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId()); + if (!sortMap.containsKey(key)) { + sortMap.put(key, new ArrayList<>(4)); + } +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java +index 9a5fa89ab..13094331e 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java +@@ -282,6 +282,10 @@ public class ExtraInfoUtil { + return (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) ? RETRY_TOPIC : NORMAL_TOPIC) + "@" + key; + } + ++ public static String getStartOffsetInfoMapKey(String topic, String popCk, long key) { ++ return ((topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || popCk != null) ? RETRY_TOPIC : NORMAL_TOPIC) + "@" + key; ++ } ++ + public static String getQueueOffsetKeyValueKey(long queueId, long queueOffset) { + return QUEUE_OFFSET + queueId + "%" + queueOffset; + } +-- +2.32.0.windows.2 + + +From a9c0b43f7f6ce5acfc4f2f3069553071fa93dfee Mon Sep 17 00:00:00 2001 +From: yx9o +Date: Wed, 16 Aug 2023 18:45:00 +0800 +Subject: [PATCH 11/12] [ISSUE #7192] Correct typos (#7193) + +--- + .../tools/command/consumer/ConsumerProgressSubCommand.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +index f51a24673..97125b854 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +@@ -54,7 +54,7 @@ public class ConsumerProgressSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Query consumers's progress, speed"; ++ return "Query consumer's progress, speed."; + } + + @Override +-- +2.32.0.windows.2 + + +From 5a3de926b816db5a121c1d788430072a3bc942ae Mon Sep 17 00:00:00 2001 +From: Zhouxiang Zhan +Date: Wed, 16 Aug 2023 20:52:53 +0800 +Subject: [PATCH 12/12] Optimize updateSubscription check exist loop (#7190) + +--- + .../broker/client/ConsumerGroupInfo.java | 17 ++++++----------- + 1 file changed, 6 insertions(+), 11 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java +index 867b9c720..1ea58c125 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java +@@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.client; + + import io.netty.channel.Channel; + import java.util.ArrayList; ++import java.util.HashSet; + import java.util.Iterator; + import java.util.List; + import java.util.Map.Entry; +@@ -172,7 +173,7 @@ public class ConsumerGroupInfo { + */ + public boolean updateSubscription(final Set subList) { + boolean updated = false; +- ++ Set topicSet = new HashSet<>(); + for (SubscriptionData sub : subList) { + SubscriptionData old = this.subscriptionTable.get(sub.getTopic()); + if (old == null) { +@@ -194,22 +195,16 @@ public class ConsumerGroupInfo { + + this.subscriptionTable.put(sub.getTopic(), sub); + } ++ // Add all new topics to the HashSet ++ topicSet.add(sub.getTopic()); + } + + Iterator> it = this.subscriptionTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String oldTopic = next.getKey(); +- +- boolean exist = false; +- for (SubscriptionData sub : subList) { +- if (sub.getTopic().equals(oldTopic)) { +- exist = true; +- break; +- } +- } +- +- if (!exist) { ++ // Check HashSet with O(1) time complexity ++ if (!topicSet.contains(oldTopic)) { + log.warn("subscription changed, group: {} remove topic {} {}", + this.groupName, + oldTopic, +-- +2.32.0.windows.2 + diff --git a/patch011-backport-optimize-config b/patch011-backport-optimize-config new file mode 100644 index 000000000..fc8e2eceb --- /dev/null +++ b/patch011-backport-optimize-config @@ -0,0 +1,1390 @@ +From 50d1050437ed8748f86ee50261b50a1e1f63162e Mon Sep 17 00:00:00 2001 +From: Jixiang Jin +Date: Wed, 16 Aug 2023 21:15:00 +0800 +Subject: [PATCH 1/7] To config the cardinalityLimit for openTelemetry metrics + exporting and fix logging config for metrics (#7196) + +--- + WORKSPACE | 14 +++--- + .../broker/metrics/BrokerMetricsManager.java | 47 ++++++++++++++----- + .../broker/metrics/PopMetricsManager.java | 11 +++-- + .../src/main/resources/rmq.broker.logback.xml | 17 ++++--- + .../apache/rocketmq/common/BrokerConfig.java | 9 ++++ + .../metrics/ControllerMetricsManager.java | 6 +-- + pom.xml | 4 +- + .../metrics/RemotingMetricsManager.java | 10 ++-- + .../rocketmq/store/DefaultMessageStore.java | 24 +++++----- + .../apache/rocketmq/store/MessageStore.java | 6 +-- + .../metrics/DefaultStoreMetricsManager.java | 4 +- + .../plugin/AbstractPluginMessageStore.java | 6 +-- + .../tieredstore/TieredMessageStore.java | 6 +-- + .../metrics/TieredStoreMetricsManager.java | 23 +++++---- + 14 files changed, 110 insertions(+), 77 deletions(-) + +diff --git a/WORKSPACE b/WORKSPACE +index a8a0aafe9..3126f2d1d 100644 +--- a/WORKSPACE ++++ b/WORKSPACE +@@ -88,14 +88,14 @@ maven_install( + "io.grpc:grpc-api:1.47.0", + "io.grpc:grpc-testing:1.47.0", + "org.springframework:spring-core:5.3.26", +- "io.opentelemetry:opentelemetry-exporter-otlp:1.19.0", +- "io.opentelemetry:opentelemetry-exporter-prometheus:1.19.0-alpha", +- "io.opentelemetry:opentelemetry-exporter-logging:1.19.0", +- "io.opentelemetry:opentelemetry-sdk:1.19.0", ++ "io.opentelemetry:opentelemetry-exporter-otlp:1.29.0", ++ "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha", ++ "io.opentelemetry:opentelemetry-exporter-logging:1.29.0", ++ "io.opentelemetry:opentelemetry-sdk:1.29.0", + "com.squareup.okio:okio-jvm:3.0.0", +- "io.opentelemetry:opentelemetry-api:1.19.0", +- "io.opentelemetry:opentelemetry-sdk-metrics:1.19.0", +- "io.opentelemetry:opentelemetry-sdk-common:1.19.0", ++ "io.opentelemetry:opentelemetry-api:1.29.0", ++ "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0", ++ "io.opentelemetry:opentelemetry-sdk-common:1.29.0", + "io.github.aliyunmq:rocketmq-slf4j-api:1.0.0", + "io.github.aliyunmq:rocketmq-logback-classic:1.0.0", + "org.slf4j:jul-to-slf4j:2.0.6", +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +index f0b76107e..6af5afc14 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +@@ -34,8 +34,10 @@ import io.opentelemetry.sdk.metrics.InstrumentType; + import io.opentelemetry.sdk.metrics.SdkMeterProvider; + import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; + import io.opentelemetry.sdk.metrics.View; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + import io.opentelemetry.sdk.metrics.data.AggregationTemporality; + import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; ++import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; + import io.opentelemetry.sdk.resources.Resource; + import java.util.ArrayList; + import java.util.Arrays; +@@ -361,22 +363,45 @@ public class BrokerMetricsManager { + .setType(InstrumentType.HISTOGRAM) + .setName(HISTOGRAM_MESSAGE_SIZE) + .build(); +- View messageSizeView = View.builder() +- .setAggregation(Aggregation.explicitBucketHistogram(messageSizeBuckets)) +- .build(); +- providerBuilder.registerView(messageSizeSelector, messageSizeView); +- +- for (Pair selectorViewPair : RemotingMetricsManager.getMetricsView()) { +- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2()); ++ ViewBuilder messageSizeViewBuilder = View.builder() ++ .setAggregation(Aggregation.explicitBucketHistogram(messageSizeBuckets)); ++ // To config the cardinalityLimit for openTelemetry metrics exporting. ++ SdkMeterProviderUtil.setCardinalityLimit(messageSizeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); ++ providerBuilder.registerView(messageSizeSelector, messageSizeViewBuilder.build()); ++ ++ for (Pair selectorViewPair : RemotingMetricsManager.getMetricsView()) { ++ ViewBuilder viewBuilder = selectorViewPair.getObject2(); ++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); ++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build()); + } + +- for (Pair selectorViewPair : messageStore.getMetricsView()) { +- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2()); ++ for (Pair selectorViewPair : messageStore.getMetricsView()) { ++ ViewBuilder viewBuilder = selectorViewPair.getObject2(); ++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); ++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build()); + } + +- for (Pair selectorViewPair : PopMetricsManager.getMetricsView()) { +- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2()); ++ for (Pair selectorViewPair : PopMetricsManager.getMetricsView()) { ++ ViewBuilder viewBuilder = selectorViewPair.getObject2(); ++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); ++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build()); + } ++ ++ // default view builder for all counter. ++ InstrumentSelector defaultCounterSelector = InstrumentSelector.builder() ++ .setType(InstrumentType.COUNTER) ++ .build(); ++ ViewBuilder defaultCounterViewBuilder = View.builder().setDescription("default view for counter."); ++ SdkMeterProviderUtil.setCardinalityLimit(defaultCounterViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); ++ providerBuilder.registerView(defaultCounterSelector, defaultCounterViewBuilder.build()); ++ ++ //default view builder for all observable gauge. ++ InstrumentSelector defaultGaugeSelector = InstrumentSelector.builder() ++ .setType(InstrumentType.OBSERVABLE_GAUGE) ++ .build(); ++ ViewBuilder defaultGaugeViewBuilder = View.builder().setDescription("default view for gauge."); ++ SdkMeterProviderUtil.setCardinalityLimit(defaultGaugeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); ++ providerBuilder.registerView(defaultGaugeSelector, defaultGaugeViewBuilder.build()); + } + + private void initStatsMetrics() { +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java +index 463371d7e..2de220da1 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java +@@ -27,6 +27,7 @@ import io.opentelemetry.sdk.metrics.Aggregation; + import io.opentelemetry.sdk.metrics.InstrumentSelector; + import io.opentelemetry.sdk.metrics.InstrumentType; + import io.opentelemetry.sdk.metrics.View; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.time.Duration; + import java.util.Arrays; + import java.util.List; +@@ -63,7 +64,7 @@ public class PopMetricsManager { + private static LongCounter popReviveGetTotal = new NopLongCounter(); + private static LongCounter popReviveRetryMessageTotal = new NopLongCounter(); + +- public static List> getMetricsView() { ++ public static List> getMetricsView() { + List rpcCostTimeBuckets = Arrays.asList( + (double) Duration.ofMillis(1).toMillis(), + (double) Duration.ofMillis(10).toMillis(), +@@ -76,10 +77,10 @@ public class PopMetricsManager { + .setType(InstrumentType.HISTOGRAM) + .setName(HISTOGRAM_POP_BUFFER_SCAN_TIME_CONSUME) + .build(); +- View popBufferScanTimeConsumeView = View.builder() +- .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)) +- .build(); +- return Lists.newArrayList(new Pair<>(popBufferScanTimeConsumeSelector, popBufferScanTimeConsumeView)); ++ ViewBuilder popBufferScanTimeConsumeViewBuilder = View.builder() ++ .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)); ++ ++ return Lists.newArrayList(new Pair<>(popBufferScanTimeConsumeSelector, popBufferScanTimeConsumeViewBuilder)); + } + + public static void initMetrics(Meter meter, BrokerController brokerController, +diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml +index 7d49f6664..3c51e59d4 100644 +--- a/broker/src/main/resources/rmq.broker.logback.xml ++++ b/broker/src/main/resources/rmq.broker.logback.xml +@@ -559,27 +559,27 @@ + + + +- ++ + + brokerContainerLogDir + ${file.separator} + + +- + +- ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_metric.log ++ ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_metrics.log + + true + + +- ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_metric.%i.log.gz ++ ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_metrics.%i.log.gz + + 1 +- 10 ++ 3 + + +- 500MB ++ 512MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n +@@ -588,6 +588,9 @@ + + + ++ ++ ++ + + + +@@ -670,7 +673,7 @@ + + + +- ++ + + + +diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +index 99a5db5ad..45d26b29c 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +@@ -350,6 +350,7 @@ public class BrokerConfig extends BrokerIdentity { + + private MetricsExporterType metricsExporterType = MetricsExporterType.DISABLE; + ++ private int metricsOtelCardinalityLimit = 50 * 1000; + private String metricsGrpcExporterTarget = ""; + private String metricsGrpcExporterHeader = ""; + private long metricGrpcExporterTimeOutInMills = 3 * 1000; +@@ -1531,6 +1532,14 @@ public class BrokerConfig extends BrokerIdentity { + this.metricsExporterType = MetricsExporterType.valueOf(metricsExporterType); + } + ++ public int getMetricsOtelCardinalityLimit() { ++ return metricsOtelCardinalityLimit; ++ } ++ ++ public void setMetricsOtelCardinalityLimit(int metricsOtelCardinalityLimit) { ++ this.metricsOtelCardinalityLimit = metricsOtelCardinalityLimit; ++ } ++ + public String getMetricsGrpcExporterTarget() { + return metricsGrpcExporterTarget; + } +diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +index 9b30a3b43..650740bcc 100644 +--- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +@@ -203,7 +203,7 @@ public class ControllerMetricsManager { + 10 * s + ); + +- View latecyView = View.builder() ++ View latencyView = View.builder() + .setAggregation(Aggregation.explicitBucketHistogram(latencyBuckets)) + .build(); + +@@ -217,8 +217,8 @@ public class ControllerMetricsManager { + .setName(HISTOGRAM_DLEDGER_OP_LATENCY) + .build(); + +- providerBuilder.registerView(requestLatencySelector, latecyView); +- providerBuilder.registerView(dLedgerOpLatencySelector, latecyView); ++ providerBuilder.registerView(requestLatencySelector, latencyView); ++ providerBuilder.registerView(dLedgerOpLatencySelector, latencyView); + } + + private void initMetric(Meter meter) { +diff --git a/pom.xml b/pom.xml +index 3a08d75f2..9f0b3eb96 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -133,8 +133,8 @@ + 2.9.3 + 5.3.27 + 3.0.0 +- 1.26.0 +- 1.26.0-alpha ++ 1.29.0 ++ 1.29.0-alpha + 2.0.6 + 2.20.29 + 1.0.3 +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java b/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java +index 34136f94f..2e0d70856 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java +@@ -26,6 +26,7 @@ import io.opentelemetry.sdk.metrics.Aggregation; + import io.opentelemetry.sdk.metrics.InstrumentSelector; + import io.opentelemetry.sdk.metrics.InstrumentType; + import io.opentelemetry.sdk.metrics.View; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.time.Duration; + import java.util.Arrays; + import java.util.List; +@@ -61,7 +62,7 @@ public class RemotingMetricsManager { + .build(); + } + +- public static List> getMetricsView() { ++ public static List> getMetricsView() { + List rpcCostTimeBuckets = Arrays.asList( + (double) Duration.ofMillis(1).toMillis(), + (double) Duration.ofMillis(3).toMillis(), +@@ -77,10 +78,9 @@ public class RemotingMetricsManager { + .setType(InstrumentType.HISTOGRAM) + .setName(HISTOGRAM_RPC_LATENCY) + .build(); +- View view = View.builder() +- .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)) +- .build(); +- return Lists.newArrayList(new Pair<>(selector, view)); ++ ViewBuilder viewBuilder = View.builder() ++ .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)); ++ return Lists.newArrayList(new Pair<>(selector, viewBuilder)); + } + + public static String getWriteAndFlushResult(Future future) { +diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +index 25e4a166f..6115ead59 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +@@ -22,7 +22,7 @@ import io.openmessaging.storage.dledger.entry.DLedgerEntry; + import io.opentelemetry.api.common.AttributesBuilder; + import io.opentelemetry.api.metrics.Meter; + import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.View; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.io.File; + import java.io.IOException; + import java.io.RandomAccessFile; +@@ -42,23 +42,24 @@ import java.util.Map; + import java.util.Objects; + import java.util.Optional; + import java.util.Set; +-import java.util.concurrent.LinkedBlockingQueue; +-import java.util.concurrent.TimeUnit; +-import java.util.concurrent.TimeoutException; +-import java.util.concurrent.ExecutionException; +-import java.util.concurrent.Executors; +-import java.util.concurrent.ExecutorService; +-import java.util.concurrent.ThreadPoolExecutor; +-import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.CompletableFuture; +-import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentLinkedQueue; ++import java.util.concurrent.ConcurrentMap; ++import java.util.concurrent.ExecutionException; ++import java.util.concurrent.ExecutorService; ++import java.util.concurrent.Executors; ++import java.util.concurrent.LinkedBlockingQueue; ++import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.ThreadPoolExecutor; ++import java.util.concurrent.TimeUnit; ++import java.util.concurrent.TimeoutException; + import java.util.concurrent.atomic.AtomicInteger; + import java.util.concurrent.atomic.AtomicLong; + import java.util.function.Supplier; + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.common.AbstractBrokerRunnable; ++import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.BrokerConfig; + import org.apache.rocketmq.common.BrokerIdentity; + import org.apache.rocketmq.common.MixAll; +@@ -82,7 +83,6 @@ import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.common.utils.CleanupPolicyUtils; + import org.apache.rocketmq.common.utils.QueueTypeUtils; + import org.apache.rocketmq.common.utils.ServiceProvider; +-import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; +@@ -3268,7 +3268,7 @@ public class DefaultMessageStore implements MessageStore { + } + + @Override +- public List> getMetricsView() { ++ public List> getMetricsView() { + return DefaultStoreMetricsManager.getMetricsView(); + } + +diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +index 31bbb907f..989cbbe31 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +@@ -19,8 +19,7 @@ package org.apache.rocketmq.store; + import io.opentelemetry.api.common.AttributesBuilder; + import io.opentelemetry.api.metrics.Meter; + import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.View; +- ++import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.nio.ByteBuffer; + import java.util.HashMap; + import java.util.LinkedList; +@@ -28,7 +27,6 @@ import java.util.List; + import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.function.Supplier; +- + import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.SystemClock; +@@ -964,7 +962,7 @@ public interface MessageStore { + * + * @return List of metrics selector and view pair + */ +- List> getMetricsView(); ++ List> getMetricsView(); + + /** + * Init store metrics +diff --git a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java +index ff87f6369..45a6bbc68 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java ++++ b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java +@@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongCounter; + import io.opentelemetry.api.metrics.Meter; + import io.opentelemetry.api.metrics.ObservableLongGauge; + import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.View; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.io.File; + import java.util.List; + import java.util.function.Supplier; +@@ -69,7 +69,7 @@ public class DefaultStoreMetricsManager { + public static LongCounter timerDequeueTotal = new NopLongCounter(); + public static LongCounter timerEnqueueTotal = new NopLongCounter(); + +- public static List> getMetricsView() { ++ public static List> getMetricsView() { + return Lists.newArrayList(); + } + +diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +index 25e947512..ab9fc6da7 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +@@ -20,8 +20,7 @@ package org.apache.rocketmq.store.plugin; + import io.opentelemetry.api.common.AttributesBuilder; + import io.opentelemetry.api.metrics.Meter; + import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.View; +- ++import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.nio.ByteBuffer; + import java.util.HashMap; + import java.util.LinkedList; +@@ -29,7 +28,6 @@ import java.util.List; + import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.function.Supplier; +- + import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.SystemClock; + import org.apache.rocketmq.common.message.MessageExt; +@@ -643,7 +641,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + } + + @Override +- public List> getMetricsView() { ++ public List> getMetricsView() { + return next.getMetricsView(); + } + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +index ced1fb818..5240ac8e9 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +@@ -21,7 +21,7 @@ import io.opentelemetry.api.common.Attributes; + import io.opentelemetry.api.common.AttributesBuilder; + import io.opentelemetry.api.metrics.Meter; + import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.View; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.util.List; + import java.util.Set; + import java.util.concurrent.CompletableFuture; +@@ -352,8 +352,8 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + } + + @Override +- public List> getMetricsView() { +- List> res = super.getMetricsView(); ++ public List> getMetricsView() { ++ List> res = super.getMetricsView(); + res.addAll(TieredStoreMetricsManager.getMetricsView()); + return res; + } +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java +index 3ca0fb614..d8a07f0a7 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java +@@ -27,6 +27,7 @@ import io.opentelemetry.sdk.metrics.Aggregation; + import io.opentelemetry.sdk.metrics.InstrumentSelector; + import io.opentelemetry.sdk.metrics.InstrumentType; + import io.opentelemetry.sdk.metrics.View; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.util.ArrayList; + import java.util.Arrays; + import java.util.HashMap; +@@ -101,8 +102,8 @@ public class TieredStoreMetricsManager { + public static ObservableLongGauge storageSize = new NopObservableLongGauge(); + public static ObservableLongGauge storageMessageReserveTime = new NopObservableLongGauge(); + +- public static List> getMetricsView() { +- ArrayList> res = new ArrayList<>(); ++ public static List> getMetricsView() { ++ ArrayList> res = new ArrayList<>(); + + InstrumentSelector providerRpcLatencySelector = InstrumentSelector.builder() + .setType(InstrumentType.HISTOGRAM) +@@ -114,10 +115,9 @@ public class TieredStoreMetricsManager { + .setName(HISTOGRAM_API_LATENCY) + .build(); + +- View rpcLatencyView = View.builder() ++ ViewBuilder rpcLatencyViewBuilder = View.builder() + .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d, 3d, 5d, 7d, 10d, 100d, 200d, 400d, 600d, 800d, 1d * 1000, 1d * 1500, 1d * 3000))) +- .setDescription("tiered_store_rpc_latency_view") +- .build(); ++ .setDescription("tiered_store_rpc_latency_view"); + + InstrumentSelector uploadBufferSizeSelector = InstrumentSelector.builder() + .setType(InstrumentType.HISTOGRAM) +@@ -129,15 +129,14 @@ public class TieredStoreMetricsManager { + .setName(HISTOGRAM_DOWNLOAD_BYTES) + .build(); + +- View bufferSizeView = View.builder() ++ ViewBuilder bufferSizeViewBuilder = View.builder() + .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d * TieredStoreUtil.KB, 10d * TieredStoreUtil.KB, 100d * TieredStoreUtil.KB, 1d * TieredStoreUtil.MB, 10d * TieredStoreUtil.MB, 32d * TieredStoreUtil.MB, 50d * TieredStoreUtil.MB, 100d * TieredStoreUtil.MB))) +- .setDescription("tiered_store_buffer_size_view") +- .build(); ++ .setDescription("tiered_store_buffer_size_view"); + +- res.add(new Pair<>(rpcLatencySelector, rpcLatencyView)); +- res.add(new Pair<>(providerRpcLatencySelector, rpcLatencyView)); +- res.add(new Pair<>(uploadBufferSizeSelector, bufferSizeView)); +- res.add(new Pair<>(downloadBufferSizeSelector, bufferSizeView)); ++ res.add(new Pair<>(rpcLatencySelector, rpcLatencyViewBuilder)); ++ res.add(new Pair<>(providerRpcLatencySelector, rpcLatencyViewBuilder)); ++ res.add(new Pair<>(uploadBufferSizeSelector, bufferSizeViewBuilder)); ++ res.add(new Pair<>(downloadBufferSizeSelector, bufferSizeViewBuilder)); + return res; + } + +-- +2.32.0.windows.2 + + +From a4bcc2a74d8bec9c9d34565536e87df06e0b11c1 Mon Sep 17 00:00:00 2001 +From: Ziyi Tan +Date: Thu, 17 Aug 2023 13:53:48 +0800 +Subject: [PATCH 2/7] [ISSUE #7178] refresh metadata after broker startup + +Signed-off-by: Ziy1-Tan +--- + .../rocketmq/broker/BrokerController.java | 24 +++++++++---------- + 1 file changed, 12 insertions(+), 12 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index 30b1d2299..13f9d002b 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -663,7 +663,7 @@ public class BrokerController { + BrokerController.this.getSlaveSynchronize().syncAll(); + lastSyncTimeMs = System.currentTimeMillis(); + } +- ++ + //timer checkpoint, latency-sensitive, so sync it more frequently + if (messageStoreConfig.isTimerWheelEnable()) { + BrokerController.this.getSlaveSynchronize().syncTimerCheckPoint(); +@@ -698,17 +698,6 @@ public class BrokerController { + + initializeBrokerScheduledTasks(); + +- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { +- @Override +- public void run() { +- try { +- BrokerController.this.brokerOuterAPI.refreshMetadata(); +- } catch (Exception e) { +- LOG.error("ScheduledTask refresh metadata exception", e); +- } +- } +- }, 10, 5, TimeUnit.SECONDS); +- + if (this.brokerConfig.getNamesrvAddr() != null) { + this.updateNamesrvAddr(); + LOG.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr()); +@@ -1682,6 +1671,17 @@ public class BrokerController { + if (brokerConfig.isSkipPreOnline()) { + startServiceWithoutCondition(); + } ++ ++ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { ++ @Override ++ public void run() { ++ try { ++ BrokerController.this.brokerOuterAPI.refreshMetadata(); ++ } catch (Exception e) { ++ LOG.error("ScheduledTask refresh metadata exception", e); ++ } ++ } ++ }, 10, 5, TimeUnit.SECONDS); + } + + protected void scheduleSendHeartbeat() { +-- +2.32.0.windows.2 + + +From 3df1b9232af99944cb3d4d4d2d00c5a85cd3b57d Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Thu, 17 Aug 2023 13:59:04 +0800 +Subject: [PATCH 3/7] [ISSUE #7201] Remove the DefaultMessageStore.class + dependency in TransientStorePool + +Co-authored-by: guyinyou +--- + .../rocketmq/store/AllocateMappedFileService.java | 6 +++--- + .../apache/rocketmq/store/DefaultMessageStore.java | 7 +++++-- + .../apache/rocketmq/store/TransientStorePool.java | 13 ++++--------- + 3 files changed, 12 insertions(+), 14 deletions(-) + +diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java +index dca7d5325..c8420fea1 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java ++++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java +@@ -55,7 +55,7 @@ public class AllocateMappedFileService extends ServiceThread { + if (this.messageStore.isTransientStorePoolEnable()) { + if (this.messageStore.getMessageStoreConfig().isFastFailIfNoBufferInStorePool() + && BrokerRole.SLAVE != this.messageStore.getMessageStoreConfig().getBrokerRole()) { //if broker is slave, don't fast fail even no buffer in pool +- canSubmitRequests = this.messageStore.getTransientStorePool().availableBufferNums() - this.requestQueue.size(); ++ canSubmitRequests = this.messageStore.remainTransientStoreBufferNumbs() - this.requestQueue.size(); + } + } + +@@ -65,7 +65,7 @@ public class AllocateMappedFileService extends ServiceThread { + if (nextPutOK) { + if (canSubmitRequests <= 0) { + log.warn("[NOTIFYME]TransientStorePool is not enough, so create mapped file error, " + +- "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums()); ++ "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs()); + this.requestTable.remove(nextFilePath); + return null; + } +@@ -81,7 +81,7 @@ public class AllocateMappedFileService extends ServiceThread { + if (nextNextPutOK) { + if (canSubmitRequests <= 0) { + log.warn("[NOTIFYME]TransientStorePool is not enough, so skip preallocate mapped file, " + +- "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums()); ++ "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs()); + this.requestTable.remove(nextNextFilePath); + } else { + boolean offerOK = this.requestQueue.offer(nextNextReq); +diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +index 6115ead59..f2a54ddf6 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +@@ -250,7 +250,7 @@ public class DefaultMessageStore implements MessageStore { + this.reputMessageService = new ConcurrentReputMessageService(); + } + +- this.transientStorePool = new TransientStorePool(this); ++ this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog()); + + this.scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); +@@ -1983,7 +1983,10 @@ public class DefaultMessageStore implements MessageStore { + } + + public int remainTransientStoreBufferNumbs() { +- return this.transientStorePool.availableBufferNums(); ++ if (this.isTransientStorePoolEnable()) { ++ return this.transientStorePool.availableBufferNums(); ++ } ++ return Integer.MAX_VALUE; + } + + @Override +diff --git a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java +index 8c1a5338b..0d42ee69e 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java ++++ b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java +@@ -33,13 +33,11 @@ public class TransientStorePool { + private final int poolSize; + private final int fileSize; + private final Deque availableBuffers; +- private final DefaultMessageStore messageStore; + private volatile boolean isRealCommit = true; + +- public TransientStorePool(final DefaultMessageStore messageStore) { +- this.messageStore = messageStore; +- this.poolSize = messageStore.getMessageStoreConfig().getTransientStorePoolSize(); +- this.fileSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); ++ public TransientStorePool(final int poolSize, final int fileSize) { ++ this.poolSize = poolSize; ++ this.fileSize = fileSize; + this.availableBuffers = new ConcurrentLinkedDeque<>(); + } + +@@ -81,10 +79,7 @@ public class TransientStorePool { + } + + public int availableBufferNums() { +- if (messageStore.isTransientStorePoolEnable()) { +- return availableBuffers.size(); +- } +- return Integer.MAX_VALUE; ++ return availableBuffers.size(); + } + + public boolean isRealCommit() { +-- +2.32.0.windows.2 + + +From 2b93e1e32fd458d9df2091e89ea259ddd4d54061 Mon Sep 17 00:00:00 2001 +From: iamgd67 +Date: Thu, 17 Aug 2023 15:31:14 +0800 +Subject: [PATCH 4/7] Update mqbroker to use runbroker.sh instead of + runserver.sh when use --enable-proxy (#7150) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Update mqbroker to use runbroker.sh instead of runserver.sh when enabling `--enable-proxy` +this allow JVM `heap` and `gc` configuration using broker's settings instead of other common serverices'(proxy,namenode, etc). +our main purpose, like the filename `mqbroker` suggest, is to start broker (which embeds a proxy), so use broker's config is reasonable + +chinese version +mqbroker的--enable-proxy选项是启动内嵌了proxy的broker，而不是内嵌broker的proxy，而且broker的工作量和重要程度大于proxy，所以使用broker的gc和heap配置更合适 +--- + distribution/bin/mqbroker | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/distribution/bin/mqbroker b/distribution/bin/mqbroker +index 3758ed597..35eb93c44 100644 +--- a/distribution/bin/mqbroker ++++ b/distribution/bin/mqbroker +@@ -68,11 +68,11 @@ if [ "$enable_proxy" = true ]; then + if [ "$broker_config" != "" ]; then + args_for_proxy=${args_for_proxy}" -bc "${broker_config} + fi +- sh ${ROCKETMQ_HOME}/bin/runserver.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy} ++ sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy} + else + args_for_broker=$other_args + if [ "$broker_config" != "" ]; then + args_for_broker=${args_for_broker}" -c "${broker_config} + fi + sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.broker.logback.xml org.apache.rocketmq.broker.BrokerStartup ${args_for_broker} +-fi +\ No newline at end of file ++fi +-- +2.32.0.windows.2 + + +From 05e7cde610255ed9410fffb0f153efe7c2c8a326 Mon Sep 17 00:00:00 2001 +From: yao-wenbin +Date: Fri, 18 Aug 2023 09:49:59 +0800 +Subject: [PATCH 5/7] [ISSUE #7042] maven-compile job failed, Because TlsTest's + serverRejectsSSLClient test case will throw TooLongFrameException (#7179) + +--- + .../remoting/netty/NettyRemotingServer.java | 2 +- + .../java/org/apache/rocketmq/remoting/TlsTest.java | 14 ++++++++++++-- + 2 files changed, 13 insertions(+), 3 deletions(-) + +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +index 90e358ce3..17f138f86 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +@@ -502,7 +502,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti + case DISABLED: + ctx.close(); + log.warn("Clients intend to establish an SSL connection while this server is running in SSL disabled mode"); +- break; ++ throw new UnsupportedOperationException("The NettyRemotingServer in SSL disabled mode doesn't support ssl client"); + case PERMISSIVE: + case ENFORCING: + if (null != sslContext) { +diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java +index de7edbbfb..a4890d73d 100644 +--- a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java ++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java +@@ -144,8 +144,13 @@ public class TlsTest { + tlsClientKeyPath = ""; + tlsClientCertPath = ""; + clientConfig.setUseTLS(false); +- } else if ("serverRejectsSSLClient".equals(name.getMethodName())) { ++ } else if ("disabledServerRejectsSSLClient".equals(name.getMethodName())) { + tlsMode = TlsMode.DISABLED; ++ } else if ("disabledServerAcceptUnAuthClient".equals(name.getMethodName())) { ++ tlsMode = TlsMode.DISABLED; ++ tlsClientKeyPath = ""; ++ tlsClientCertPath = ""; ++ clientConfig.setUseTLS(false); + } else if ("reloadSslContextForServer".equals(name.getMethodName())) { + tlsClientAuthServer = false; + tlsServerNeedClientAuth = "none"; +@@ -211,7 +216,7 @@ public class TlsTest { + } + + @Test +- public void serverRejectsSSLClient() throws Exception { ++ public void disabledServerRejectsSSLClient() throws Exception { + try { + RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 5); + failBecauseExceptionWasNotThrown(RemotingSendRequestException.class); +@@ -219,6 +224,11 @@ public class TlsTest { + } + } + ++ @Test ++ public void disabledServerAcceptUnAuthClient() throws Exception { ++ requestThenAssertResponse(); ++ } ++ + /** + * Tests that a server configured to require client authentication refuses to accept connections + * from a client that has an untrusted certificate. +-- +2.32.0.windows.2 + + +From 72d796f2b20b3ec6aebca8c004d9275d7c749a95 Mon Sep 17 00:00:00 2001 +From: lk +Date: Fri, 18 Aug 2023 11:55:39 +0800 +Subject: [PATCH 6/7] [ISSUE #7205] support batch ack for pop orderly (#7206) + +--- + .../broker/processor/AckMessageProcessor.java | 99 ++++++----- + .../rocketmq/client/impl/MQClientAPIImpl.java | 91 ++++++++-- + .../test/client/rmq/RMQPopClient.java | 22 +++ + .../client/consumer/pop/BasePopNormally.java | 6 + + .../test/client/consumer/pop/BatchAckIT.java | 159 ++++++++++++++++++ + 5 files changed, 322 insertions(+), 55 deletions(-) + create mode 100644 test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +index 687811409..244b459d6 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.processor; + import com.alibaba.fastjson.JSON; + import io.netty.channel.Channel; + import io.netty.channel.ChannelHandlerContext; ++import java.util.BitSet; + import org.apache.rocketmq.broker.BrokerController; + import org.apache.rocketmq.broker.metrics.PopMetricsManager; + import org.apache.rocketmq.common.KeyBuilder; +@@ -186,46 +187,7 @@ public class AckMessageProcessor implements NettyRequestProcessor { + invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfo); + + if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) { +- // order +- String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId; +- long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); +- if (ackOffset < oldOffset) { +- return; +- } +- while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) { +- } +- try { +- oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); +- if (ackOffset < oldOffset) { +- return; +- } +- long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext( +- topic, consumeGroup, +- qId, ackOffset, +- popTime); +- if (nextOffset > -1) { +- if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset( +- topic, consumeGroup, qId)) { +- this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), +- consumeGroup, topic, qId, nextOffset); +- } +- if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, +- consumeGroup, qId, invisibleTime)) { +- this.brokerController.getPopMessageProcessor().notifyMessageArriving( +- topic, consumeGroup, qId); +- } +- } else if (nextOffset == -1) { +- String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", +- lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress()); +- POP_LOGGER.warn(errorInfo); +- response.setCode(ResponseCode.MESSAGE_ILLEGAL); +- response.setRemark(errorInfo); +- return; +- } +- } finally { +- this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey); +- } +- brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount); ++ ackOrderly(topic, consumeGroup, qId, ackOffset, popTime, invisibleTime, channel, response); + return; + } + +@@ -250,17 +212,22 @@ public class AckMessageProcessor implements NettyRequestProcessor { + } + + BatchAckMsg batchAckMsg = new BatchAckMsg(); +- for (int i = 0; batchAck.getBitSet() != null && i < batchAck.getBitSet().length(); i++) { +- if (!batchAck.getBitSet().get(i)) { +- continue; ++ BitSet bitSet = batchAck.getBitSet(); ++ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { ++ if (i == Integer.MAX_VALUE) { ++ break; + } + long offset = startOffset + i; + if (offset < minOffset || offset > maxOffset) { + continue; + } +- batchAckMsg.getAckOffsetList().add(offset); ++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) { ++ ackOrderly(topic, consumeGroup, qId, offset, popTime, invisibleTime, channel, response); ++ } else { ++ batchAckMsg.getAckOffsetList().add(offset); ++ } + } +- if (batchAckMsg.getAckOffsetList().isEmpty()) { ++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE || batchAckMsg.getAckOffsetList().isEmpty()) { + return; + } + +@@ -311,4 +278,46 @@ public class AckMessageProcessor implements NettyRequestProcessor { + PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus()); + brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount); + } ++ ++ protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime, long invisibleTime, Channel channel, RemotingCommand response) { ++ String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId; ++ long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); ++ if (ackOffset < oldOffset) { ++ return; ++ } ++ while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) { ++ } ++ try { ++ oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); ++ if (ackOffset < oldOffset) { ++ return; ++ } ++ long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext( ++ topic, consumeGroup, ++ qId, ackOffset, ++ popTime); ++ if (nextOffset > -1) { ++ if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset( ++ topic, consumeGroup, qId)) { ++ this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), ++ consumeGroup, topic, qId, nextOffset); ++ } ++ if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, ++ consumeGroup, qId, invisibleTime)) { ++ this.brokerController.getPopMessageProcessor().notifyMessageArriving( ++ topic, consumeGroup, qId); ++ } ++ } else if (nextOffset == -1) { ++ String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", ++ lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress()); ++ POP_LOGGER.warn(errorInfo); ++ response.setCode(ResponseCode.MESSAGE_ILLEGAL); ++ response.setRemark(errorInfo); ++ return; ++ } ++ } finally { ++ this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey); ++ } ++ brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, 1); ++ } + } +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +index 5101ffc8e..213c26fd6 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +@@ -21,6 +21,7 @@ import java.io.UnsupportedEncodingException; + import java.nio.ByteBuffer; + import java.util.ArrayList; + import java.util.Arrays; ++import java.util.BitSet; + import java.util.Collections; + import java.util.HashMap; + import java.util.Iterator; +@@ -54,6 +55,7 @@ import org.apache.rocketmq.client.producer.SendCallback; + import org.apache.rocketmq.client.producer.SendResult; + import org.apache.rocketmq.client.producer.SendStatus; + import org.apache.rocketmq.common.AclConfig; ++import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.MQVersion; + import org.apache.rocketmq.common.MixAll; + import org.apache.rocketmq.common.Pair; +@@ -76,7 +78,8 @@ import org.apache.rocketmq.common.namesrv.NameServerUpdateCallback; + import org.apache.rocketmq.common.namesrv.TopAddressing; + import org.apache.rocketmq.common.sysflag.PullSysFlag; + import org.apache.rocketmq.common.topic.TopicValidator; +-import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.CommandCustomHeader; + import org.apache.rocketmq.remoting.InvokeCallback; + import org.apache.rocketmq.remoting.RPCHook; +@@ -101,7 +104,10 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; + import org.apache.rocketmq.remoting.protocol.ResponseCode; + import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; + import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; ++import org.apache.rocketmq.remoting.protocol.body.BatchAck; ++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; + import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; ++import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; + import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData; + import org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody; + import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo; +@@ -114,7 +120,6 @@ import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; + import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody; + import org.apache.rocketmq.remoting.protocol.body.GroupList; + import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; +-import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; + import org.apache.rocketmq.remoting.protocol.body.KVTable; + import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; + import org.apache.rocketmq.remoting.protocol.body.LockBatchResponseBody; +@@ -196,6 +201,10 @@ import org.apache.rocketmq.remoting.protocol.header.UpdateGlobalWhiteAddrsConfig + import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.ViewBrokerStatsDataRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader; ++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; ++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; ++import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader; + import org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader; +@@ -207,10 +216,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestH + import org.apache.rocketmq.remoting.protocol.header.namesrv.PutKVConfigRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader; +-import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; +-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; +-import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; + import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData; + import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; + import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +@@ -221,8 +226,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden; + import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; + import org.apache.rocketmq.remoting.rpchook.DynamicalExtFieldRPCHook; + import org.apache.rocketmq.remoting.rpchook.StreamTypeRPCHook; +-import org.apache.rocketmq.logging.org.slf4j.Logger; +-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + + import static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS; + +@@ -885,9 +888,77 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + final String addr, + final long timeOut, + final AckCallback ackCallback, +- final AckMessageRequestHeader requestHeader // ++ final AckMessageRequestHeader requestHeader ++ ) throws RemotingException, MQBrokerException, InterruptedException { ++ ackMessageAsync(addr, timeOut, ackCallback, requestHeader, null); ++ } ++ ++ public void batchAckMessageAsync( ++ final String addr, ++ final long timeOut, ++ final AckCallback ackCallback, ++ final String topic, ++ final String consumerGroup, ++ final List extraInfoList ++ ) throws RemotingException, MQBrokerException, InterruptedException { ++ String brokerName = null; ++ Map batchAckMap = new HashMap<>(); ++ for (String extraInfo : extraInfoList) { ++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo); ++ if (brokerName == null) { ++ brokerName = ExtraInfoUtil.getBrokerName(extraInfoData); ++ } ++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + ++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" + ++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + ++ ExtraInfoUtil.getPopTime(extraInfoData); ++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { ++ BatchAck newBatchAck = new BatchAck(); ++ newBatchAck.setConsumerGroup(consumerGroup); ++ newBatchAck.setTopic(topic); ++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); ++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); ++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); ++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); ++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); ++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); ++ newBatchAck.setBitSet(new BitSet()); ++ return newBatchAck; ++ }); ++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); ++ } ++ ++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); ++ requestBody.setBrokerName(brokerName); ++ requestBody.setAcks(new ArrayList<>(batchAckMap.values())); ++ batchAckMessageAsync(addr, timeOut, ackCallback, requestBody); ++ } ++ ++ public void batchAckMessageAsync( ++ final String addr, ++ final long timeOut, ++ final AckCallback ackCallback, ++ final BatchAckMessageRequestBody requestBody + ) throws RemotingException, MQBrokerException, InterruptedException { +- final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); ++ ackMessageAsync(addr, timeOut, ackCallback, null, requestBody); ++ } ++ ++ protected void ackMessageAsync( ++ final String addr, ++ final long timeOut, ++ final AckCallback ackCallback, ++ final AckMessageRequestHeader requestHeader, ++ final BatchAckMessageRequestBody requestBody ++ ) throws RemotingException, MQBrokerException, InterruptedException { ++ RemotingCommand request; ++ if (requestHeader != null) { ++ request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); ++ } else { ++ request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); ++ if (requestBody != null) { ++ request.setBody(requestBody.encode()); ++ } ++ } + this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) { + + @Override +diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java +index 496bd6da4..09c60c0b4 100644 +--- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java ++++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java +@@ -17,6 +17,7 @@ + + package org.apache.rocketmq.test.client.rmq; + ++import java.util.List; + import java.util.concurrent.CompletableFuture; + import org.apache.rocketmq.client.ClientConfig; + import org.apache.rocketmq.client.consumer.AckCallback; +@@ -140,6 +141,27 @@ public class RMQPopClient implements MQConsumer { + return future; + } + ++ public CompletableFuture batchAckMessageAsync(String brokerAddr, String topic, String consumerGroup, ++ List extraInfoList) { ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++ this.mqClientAPI.batchAckMessageAsync(brokerAddr, DEFAULT_TIMEOUT, new AckCallback() { ++ @Override ++ public void onSuccess(AckResult ackResult) { ++ future.complete(ackResult); ++ } ++ ++ @Override ++ public void onException(Throwable e) { ++ future.completeExceptionally(e); ++ } ++ }, topic, consumerGroup, extraInfoList); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++ return future; ++ } ++ + public CompletableFuture changeInvisibleTimeAsync(String brokerAddr, String brokerName, String topic, + String consumerGroup, String extraInfo, long invisibleTime) { + String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo); +diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java +index 952fbe3f5..2e29b95a5 100644 +--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java ++++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java +@@ -63,4 +63,10 @@ public class BasePopNormally extends BasePop { + brokerAddr, messageQueue, invisibleTime, maxNums, group, timeout, true, + ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); + } ++ ++ protected CompletableFuture popMessageAsync(long invisibleTime, int maxNums) { ++ return client.popMessageAsync( ++ brokerAddr, messageQueue, invisibleTime, maxNums, group, 3000, false, ++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); ++ } + } +diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java +new file mode 100644 +index 000000000..ec9153ccc +--- /dev/null ++++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java +@@ -0,0 +1,159 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.test.client.consumer.pop; ++ ++import java.time.Duration; ++import java.util.ArrayList; ++import java.util.List; ++import java.util.concurrent.CompletableFuture; ++import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicInteger; ++import java.util.function.Supplier; ++import org.apache.rocketmq.client.consumer.AckResult; ++import org.apache.rocketmq.client.consumer.AckStatus; ++import org.apache.rocketmq.client.consumer.PopResult; ++import org.apache.rocketmq.client.consumer.PopStatus; ++import org.apache.rocketmq.common.attribute.CQType; ++import org.apache.rocketmq.common.attribute.TopicMessageType; ++import org.apache.rocketmq.common.constant.ConsumeInitMode; ++import org.apache.rocketmq.common.filter.ExpressionType; ++import org.apache.rocketmq.common.message.MessageConst; ++import org.apache.rocketmq.common.message.MessageExt; ++import org.apache.rocketmq.common.message.MessageQueue; ++import org.apache.rocketmq.test.base.IntegrationTestBase; ++import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; ++import org.apache.rocketmq.test.client.rmq.RMQPopClient; ++import org.apache.rocketmq.test.util.MQRandomUtils; ++import org.junit.After; ++import org.junit.Before; ++import org.junit.Test; ++ ++import static org.awaitility.Awaitility.await; ++import static org.junit.Assert.assertEquals; ++ ++public class BatchAckIT extends BasePop { ++ ++ protected String topic; ++ protected String group; ++ protected RMQNormalProducer producer = null; ++ protected RMQPopClient client = null; ++ protected String brokerAddr; ++ protected MessageQueue messageQueue; ++ ++ @Before ++ public void setUp() { ++ brokerAddr = brokerController1.getBrokerAddr(); ++ topic = MQRandomUtils.getRandomTopic(); ++ group = initConsumerGroup(); ++ IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL); ++ producer = getProducer(NAMESRV_ADDR, topic); ++ client = getRMQPopClient(); ++ messageQueue = new MessageQueue(topic, BROKER1_NAME, -1); ++ } ++ ++ @After ++ public void tearDown() { ++ shutdown(); ++ } ++ ++ @Test ++ public void testBatchAckNormallyWithPopBuffer() throws Throwable { ++ brokerController1.getBrokerConfig().setEnablePopBufferMerge(true); ++ brokerController2.getBrokerConfig().setEnablePopBufferMerge(true); ++ ++ testBatchAck(() -> { ++ try { ++ return popMessageAsync().get(); ++ } catch (Exception e) { ++ throw new RuntimeException(e); ++ } ++ }); ++ } ++ ++ @Test ++ public void testBatchAckNormallyWithOutPopBuffer() throws Throwable { ++ brokerController1.getBrokerConfig().setEnablePopBufferMerge(false); ++ brokerController2.getBrokerConfig().setEnablePopBufferMerge(false); ++ ++ testBatchAck(() -> { ++ try { ++ return popMessageAsync().get(); ++ } catch (Exception e) { ++ throw new RuntimeException(e); ++ } ++ }); ++ } ++ ++ @Test ++ public void testBatchAckOrderly() throws Throwable { ++ testBatchAck(() -> { ++ try { ++ return popMessageOrderlyAsync().get(); ++ } catch (Exception e) { ++ throw new RuntimeException(e); ++ } ++ }); ++ } ++ ++ public void testBatchAck(Supplier popResultSupplier) throws Throwable { ++ // Send 10 messages but do not ack, let them enter the retry topic ++ producer.send(10); ++ AtomicInteger firstMsgRcvNum = new AtomicInteger(); ++ await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> { ++ PopResult popResult = popResultSupplier.get(); ++ if (popResult.getPopStatus().equals(PopStatus.FOUND)) { ++ firstMsgRcvNum.addAndGet(popResult.getMsgFoundList().size()); ++ } ++ assertEquals(10, firstMsgRcvNum.get()); ++ }); ++ // sleep 6s, expect messages to enter the retry topic ++ TimeUnit.SECONDS.sleep(6); ++ ++ producer.send(20); ++ List extraInfoList = new ArrayList<>(); ++ await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> { ++ PopResult popResult = popResultSupplier.get(); ++ if (popResult.getPopStatus().equals(PopStatus.FOUND)) { ++ for (MessageExt messageExt : popResult.getMsgFoundList()) { ++ extraInfoList.add(messageExt.getProperty(MessageConst.PROPERTY_POP_CK)); ++ } ++ } ++ assertEquals(30, extraInfoList.size()); ++ }); ++ ++ AckResult ackResult = client.batchAckMessageAsync(brokerAddr, topic, group, extraInfoList).get(); ++ assertEquals(AckStatus.OK, ackResult.getStatus()); ++ ++ // sleep 6s, expected that messages that have been acked will not be re-consumed ++ TimeUnit.SECONDS.sleep(6); ++ PopResult popResult = popResultSupplier.get(); ++ assertEquals(PopStatus.POLLING_NOT_FOUND, popResult.getPopStatus()); ++ } ++ ++ private CompletableFuture popMessageAsync() { ++ return client.popMessageAsync( ++ brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false, ++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); ++ } ++ ++ private CompletableFuture popMessageOrderlyAsync() { ++ return client.popMessageAsync( ++ brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false, ++ ConsumeInitMode.MIN, true, ExpressionType.TAG, "*", null); ++ } ++} +-- +2.32.0.windows.2 + + +From cc16a1b51216e1e80c22011b8b01e060bb4af8b3 Mon Sep 17 00:00:00 2001 +From: rongtong +Date: Tue, 22 Aug 2023 10:42:25 +0800 +Subject: [PATCH 7/7] Set table reference the same object for + setSubscriptionGroupTable method (#7204) + +--- + .../broker/subscription/SubscriptionGroupManager.java | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +index 74e39c0fe..e63b93058 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +@@ -341,10 +341,7 @@ public class SubscriptionGroupManager extends ConfigManager { + + + public void setSubscriptionGroupTable(ConcurrentMap subscriptionGroupTable) { +- this.subscriptionGroupTable.clear(); +- for (String key : subscriptionGroupTable.keySet()) { +- putSubscriptionGroupConfig(subscriptionGroupTable.get(key)); +- } ++ this.subscriptionGroupTable = subscriptionGroupTable; + } + + public boolean containsSubscriptionGroup(String group) { +-- +2.32.0.windows.2 + diff --git a/patch011-backport-optimize-opentelemetry-metric-config.patch b/patch011-backport-optimize-opentelemetry-metric-config.patch new file mode 100644 index 000000000..b59d7e2c7 --- /dev/null +++ b/patch011-backport-optimize-opentelemetry-metric-config.patch @@ -0,0 +1,2081 @@ +From 744167bd01fab6821b4d5ae1794dc845153d5156 Mon Sep 17 00:00:00 2001 +From: Ziyi Tan +Date: Wed, 23 Aug 2023 08:32:17 +0800 +Subject: [PATCH 1/7] [ISSUE #7142] Add command `RocksDBConfigToJson` to + inspect rocksdb content (#7180) + +* feat: add command `RocksDBConfigToJson` to inspect rocksdb content + +Signed-off-by: Ziy1-Tan + +* refactor: fix style + +--------- + +Signed-off-by: Ziy1-Tan +Co-authored-by: Ziy1-Tan +--- + .../tools/command/MQAdminStartup.java | 2 + + .../metadata/RocksDBConfigToJsonCommand.java | 118 ++++++++++++++++++ + .../metadata/KvConfigToJsonCommandTest.java | 65 ++++++++++ + 3 files changed, 185 insertions(+) + create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java + create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java + +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +index 890125ca0..324aa1856 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +@@ -80,6 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; + import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; + import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; + import org.apache.rocketmq.tools.command.message.SendMessageCommand; ++import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; + import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; + import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; + import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; +@@ -211,6 +212,7 @@ public class MQAdminStartup { + + initCommand(new ClusterListSubCommand()); + initCommand(new TopicListSubCommand()); ++ initCommand(new RocksDBConfigToJsonCommand()); + + initCommand(new UpdateKvConfigCommand()); + initCommand(new DeleteKvConfigCommand()); +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +new file mode 100644 +index 000000000..3053f4684 +--- /dev/null ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +@@ -0,0 +1,118 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.tools.command.metadata; ++ ++import com.alibaba.fastjson.JSONObject; ++import org.apache.commons.cli.CommandLine; ++import org.apache.commons.cli.Option; ++import org.apache.commons.cli.Options; ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.config.RocksDBConfigManager; ++import org.apache.rocketmq.common.utils.DataConverter; ++import org.apache.rocketmq.remoting.RPCHook; ++import org.apache.rocketmq.tools.command.SubCommand; ++import org.apache.rocketmq.tools.command.SubCommandException; ++ ++import java.io.File; ++import java.util.HashMap; ++import java.util.Map; ++ ++public class RocksDBConfigToJsonCommand implements SubCommand { ++ private static final String TOPICS_JSON_CONFIG = "topics"; ++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; ++ ++ @Override ++ public String commandName() { ++ return "rocksDBConfigToJson"; ++ } ++ ++ @Override ++ public String commandDesc() { ++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; ++ } ++ ++ @Override ++ public Options buildCommandlineOptions(Options options) { ++ Option pathOption = new Option("p", "path", true, ++ "Absolute path to the metadata directory"); ++ pathOption.setRequired(true); ++ options.addOption(pathOption); ++ ++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + ++ "topics/subscriptionGroups"); ++ configTypeOption.setRequired(true); ++ options.addOption(configTypeOption); ++ ++ return options; ++ } ++ ++ @Override ++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { ++ String path = commandLine.getOptionValue("path").trim(); ++ if (StringUtils.isEmpty(path) || !new File(path).exists()) { ++ System.out.print("Rocksdb path is invalid.\n"); ++ return; ++ } ++ ++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); ++ ++ final long memTableFlushInterval = 60 * 60 * 1000L; ++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); ++ try { ++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for topics.json ++ final Map topicsJsonConfig = new HashMap<>(); ++ final Map topicConfigTable = new HashMap<>(); ++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++ final String topic = new String(key, DataConverter.charset); ++ final String topicConfig = new String(value, DataConverter.charset); ++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig); ++ topicConfigTable.put(topic, jsonObject); ++ }); ++ ++ if (isLoad) { ++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); ++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); ++ System.out.print(topicsJsonStr + "\n"); ++ return; ++ } ++ } ++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for subscriptionGroup.json ++ final Map subscriptionGroupJsonConfig = new HashMap<>(); ++ final Map subscriptionGroupTable = new HashMap<>(); ++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++ final String subscriptionGroup = new String(key, DataConverter.charset); ++ final String subscriptionGroupConfig = new String(value, DataConverter.charset); ++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); ++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); ++ }); ++ ++ if (isLoad) { ++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", ++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); ++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); ++ System.out.print(subscriptionGroupJsonStr + "\n"); ++ return; ++ } ++ } ++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); ++ } finally { ++ kvConfigManager.stop(); ++ } ++ } ++} +diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java +new file mode 100644 +index 000000000..b2f66c7b0 +--- /dev/null ++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java +@@ -0,0 +1,65 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.tools.command.metadata; ++ ++import org.apache.commons.cli.CommandLine; ++import org.apache.commons.cli.DefaultParser; ++import org.apache.commons.cli.Options; ++import org.apache.rocketmq.srvutil.ServerUtil; ++import org.apache.rocketmq.tools.command.SubCommandException; ++import org.junit.Test; ++ ++import java.io.File; ++ ++import static org.assertj.core.api.Assertions.assertThat; ++ ++public class KvConfigToJsonCommandTest { ++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; ++ ++ @Test ++ public void testExecute() throws SubCommandException { ++ { ++ String[] cases = new String[]{"topics", "subscriptionGroups"}; ++ for (String c : cases) { ++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); ++ Options options = ServerUtil.buildCommandlineOptions(new Options()); ++ String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c}; ++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, ++ cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.execute(commandLine, options, null); ++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c); ++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c); ++ } ++ } ++ // invalid cases ++ { ++ String[][] cases = new String[][]{ ++ {"-p " + BASE_PATH + "tmpPath", "-t topics"}, ++ {"-p ", "-t topics"}, ++ {"-p " + BASE_PATH + "topics", "-t invalid_type"} ++ }; ++ ++ for (String[] c : cases) { ++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); ++ Options options = ServerUtil.buildCommandlineOptions(new Options()); ++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, ++ cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.execute(commandLine, options, null); ++ } ++ } ++ } ++} +-- +2.32.0.windows.2 + + +From bdede35db365a49b211cdc249c68b0f60a3df46d Mon Sep 17 00:00:00 2001 +From: mxsm +Date: Wed, 23 Aug 2023 08:34:56 +0800 +Subject: [PATCH 2/7] [ISSUE #7124] Fix the typos in the code comments (#7125) + +--- + .../apache/rocketmq/broker/processor/ReplyMessageProcessor.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +index b2db356c8..d3bb048f7 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +@@ -234,7 +234,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor { + } else { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); +- //set to zore to avoid client decoding exception ++ //set to zero to avoid client decoding exception + responseHeader.setMsgId("0"); + responseHeader.setQueueId(queueIdInt); + responseHeader.setQueueOffset(0L); +-- +2.32.0.windows.2 + + +From 9bb73b9a38548b99ac5126c40380c3c2e7fc586e Mon Sep 17 00:00:00 2001 +From: lizhimins <707364882@qq.com> +Date: Wed, 23 Aug 2023 09:46:27 +0800 +Subject: [PATCH 3/7] [#ISSUE 7222] Bug fix and refactoring of the Indexfile in + tiered storage (#7224) + +--- + .../tieredstore/file/TieredIndexFile.java | 38 +++++++-- + .../tieredstore/file/TieredIndexFileTest.java | 84 +++++-------------- + 2 files changed, 52 insertions(+), 70 deletions(-) + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java +index 50beb01ae..eda5e0106 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java +@@ -16,6 +16,7 @@ + */ + package org.apache.rocketmq.tieredstore.file; + ++import com.google.common.annotations.VisibleForTesting; + import java.io.File; + import java.io.IOException; + import java.nio.ByteBuffer; +@@ -99,7 +100,7 @@ public class TieredIndexFile { + this::doScheduleTask, 10, 10, TimeUnit.SECONDS); + } + +- private void doScheduleTask() { ++ protected void doScheduleTask() { + try { + curFileLock.lock(); + try { +@@ -145,6 +146,11 @@ public class TieredIndexFile { + } + } + ++ @VisibleForTesting ++ public MappedFile getPreMappedFile() { ++ return preMappedFile; ++ } ++ + private void initFile() throws IOException { + curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); + initIndexFileHeader(curMappedFile); +@@ -156,19 +162,26 @@ public class TieredIndexFile { + + if (isFileSealed(curMappedFile)) { + if (preFileExists) { +- preFile.delete(); ++ if (preFile.delete()) { ++ logger.info("Pre IndexFile deleted success", preFilepath); ++ } else { ++ logger.error("Pre IndexFile deleted failed", preFilepath); ++ } + } + boolean rename = curMappedFile.renameTo(preFilepath); + if (rename) { + preMappedFile = curMappedFile; + curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); ++ initIndexFileHeader(curMappedFile); + preFileExists = true; + } + } ++ + if (preFileExists) { + synchronized (TieredIndexFile.class) { + if (inflightCompactFuture.isDone()) { +- inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(new CompactTask(storeConfig, preMappedFile, flatFile), null); ++ inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit( ++ new CompactTask(storeConfig, preMappedFile, flatFile), null); + } + } + } +@@ -261,7 +274,8 @@ public class TieredIndexFile { + } + } + +- public CompletableFuture>> queryAsync(String topic, String key, long beginTime, long endTime) { ++ public CompletableFuture>> queryAsync(String topic, String key, long beginTime, ++ long endTime) { + int hashCode = indexKeyHashMethod(buildKey(topic, key)); + int slotPosition = hashCode % maxHashSlotNum; + List fileSegmentList = flatFile.getFileListByTime(beginTime, endTime); +@@ -355,7 +369,7 @@ public class TieredIndexFile { + private final int fileMaxSize; + private MappedFile originFile; + private TieredFlatFile fileQueue; +- private final MappedFile compactFile; ++ private MappedFile compactFile; + + public CompactTask(TieredMessageStoreConfig storeConfig, MappedFile originFile, + TieredFlatFile fileQueue) throws IOException { +@@ -381,6 +395,17 @@ public class TieredIndexFile { + } catch (Throwable throwable) { + logger.error("TieredIndexFile#compactTask: compact index file failed:", throwable); + } ++ ++ try { ++ if (originFile != null) { ++ originFile.destroy(-1); ++ } ++ if (compactFile != null) { ++ compactFile.destroy(-1); ++ } ++ } catch (Throwable throwable) { ++ logger.error("TieredIndexFile#compactTask: destroy index file failed:", throwable); ++ } + } + + public void compact() { +@@ -396,6 +421,8 @@ public class TieredIndexFile { + fileQueue.commit(true); + compactFile.destroy(-1); + originFile.destroy(-1); ++ compactFile = null; ++ originFile = null; + } + + private void buildCompactFile() { +@@ -414,6 +441,7 @@ public class TieredIndexFile { + if (slotValue != -1) { + int indexTotalSize = 0; + int indexPosition = slotValue; ++ + while (indexPosition >= 0 && indexPosition < maxIndexNum) { + int indexOffset = INDEX_FILE_HEADER_SIZE + maxHashSlotNum * INDEX_FILE_HASH_SLOT_SIZE + + indexPosition * INDEX_FILE_HASH_ORIGIN_INDEX_SIZE; +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +index 7ef49578d..262d6645b 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +@@ -19,9 +19,8 @@ package org.apache.rocketmq.tieredstore.file; + import com.sun.jna.Platform; + import java.io.IOException; + import java.nio.ByteBuffer; ++import java.time.Duration; + import java.util.List; +-import java.util.concurrent.TimeUnit; +-import org.apache.commons.lang3.SystemUtils; + import org.apache.commons.lang3.tuple.Pair; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; +@@ -31,9 +30,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + import org.awaitility.Awaitility; + import org.junit.After; + import org.junit.Assert; +-import org.junit.Assume; + import org.junit.Before; +-import org.junit.Ignore; + import org.junit.Test; + + public class TieredIndexFileTest { +@@ -45,11 +42,12 @@ public class TieredIndexFileTest { + @Before + public void setUp() { + storeConfig = new TieredMessageStoreConfig(); ++ storeConfig.setBrokerName("IndexFileBroker"); + storeConfig.setStorePathRootDir(storePath); +- storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment"); +- storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2); +- storeConfig.setTieredStoreIndexFileMaxIndexNum(3); +- mq = new MessageQueue("TieredIndexFileTest", storeConfig.getBrokerName(), 1); ++ storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment"); ++ storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5); ++ storeConfig.setTieredStoreIndexFileMaxIndexNum(20); ++ mq = new MessageQueue("IndexFileTest", storeConfig.getBrokerName(), 1); + TieredStoreUtil.getMetadataStore(storeConfig); + TieredStoreExecutor.init(); + } +@@ -61,77 +59,33 @@ public class TieredIndexFileTest { + TieredStoreExecutor.shutdown(); + } + +- @Ignore + @Test + public void testAppendAndQuery() throws IOException, ClassNotFoundException, NoSuchMethodException { + if (Platform.isWindows()) { + return; + } + +- // skip this test on windows +- Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); +- + TieredFileAllocator fileQueueFactory = new TieredFileAllocator(storeConfig); + TieredIndexFile indexFile = new TieredIndexFile(fileQueueFactory, storePath); ++ + indexFile.append(mq, 0, "key3", 3, 300, 1000); + indexFile.append(mq, 0, "key2", 2, 200, 1100); + indexFile.append(mq, 0, "key1", 1, 100, 1200); + +- Awaitility.waitAtMost(5, TimeUnit.SECONDS) +- .until(() -> { +- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); +- if (indexList.size() != 1) { +- return false; +- } +- +- ByteBuffer indexBuffer = indexList.get(0).getValue(); +- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 2, indexBuffer.remaining()); +- +- Assert.assertEquals(1, indexBuffer.getLong(4 + 4 + 4)); +- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8)); +- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +- +- Assert.assertEquals(3, indexBuffer.getLong(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4)); +- Assert.assertEquals(300, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8)); +- Assert.assertEquals(0, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8 + 4)); +- return true; +- }); +- +- indexFile.append(mq, 0, "key4", 4, 400, 1300); +- indexFile.append(mq, 0, "key4", 4, 400, 1300); +- indexFile.append(mq, 0, "key4", 4, 400, 1300); +- +- Awaitility.waitAtMost(5, TimeUnit.SECONDS) +- .until(() -> { +- List> indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1300, 1300).join(); +- if (indexList.size() != 1) { +- return false; +- } +- +- ByteBuffer indexBuffer = indexList.get(0).getValue(); +- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); +- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); +- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); +- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +- return true; +- }); +- +- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1300, 1300).join(); ++ // do not do schedule task here ++ TieredStoreExecutor.shutdown(); ++ List> indexList = ++ indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); + Assert.assertEquals(0, indexList.size()); + +- indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1200, 1300).join(); +- Assert.assertEquals(2, indexList.size()); +- +- ByteBuffer indexBuffer = indexList.get(0).getValue(); +- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); +- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); +- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); +- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++ // do compaction once ++ TieredStoreExecutor.init(); ++ storeConfig.setTieredStoreIndexFileRollingIdleInterval(0); ++ indexFile.doScheduleTask(); ++ Awaitility.await().atMost(Duration.ofSeconds(10)) ++ .until(() -> !indexFile.getPreMappedFile().getFile().exists()); + +- indexBuffer = indexList.get(1).getValue(); +- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE, indexBuffer.remaining()); +- Assert.assertEquals(2, indexBuffer.getLong(4 + 4 + 4)); +- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8)); +- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++ Assert.assertEquals(1, indexList.size()); + } + } +-- +2.32.0.windows.2 + + +From 69c26d3d29cde7b4484ecd112ab9224f9f42bf45 Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Wed, 23 Aug 2023 10:27:52 +0800 +Subject: [PATCH 4/7] [ISSUE #7228] Converge the use of some important + variables for some class + +--- + .../apache/rocketmq/store/ConsumeQueue.java | 16 ++++++------ + .../rocketmq/store/MappedFileQueue.java | 26 +++++++++++-------- + .../store/MultiPathMappedFileQueue.java | 4 +-- + 3 files changed, 24 insertions(+), 22 deletions(-) + +diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +index a0b886eb0..56bee2af3 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +@@ -145,7 +145,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + if (offset >= 0 && size > 0) { + mappedFileOffset = i + CQ_STORE_UNIT_SIZE; +- this.maxPhysicOffset = offset + size; ++ this.setMaxPhysicOffset(offset + size); + if (isExtAddr(tagsCode)) { + maxExtAddr = tagsCode; + } +@@ -409,7 +409,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + int logicFileSize = this.mappedFileSize; + +- this.maxPhysicOffset = phyOffset; ++ this.setMaxPhysicOffset(phyOffset); + long maxExtAddr = 1; + boolean shouldDeleteFile = false; + while (true) { +@@ -435,7 +435,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + mappedFile.setWrotePosition(pos); + mappedFile.setCommittedPosition(pos); + mappedFile.setFlushedPosition(pos); +- this.maxPhysicOffset = offset + size; ++ this.setMaxPhysicOffset(offset + size); + // This maybe not take effect, when not every consume queue has extend file. + if (isExtAddr(tagsCode)) { + maxExtAddr = tagsCode; +@@ -453,7 +453,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + mappedFile.setWrotePosition(pos); + mappedFile.setCommittedPosition(pos); + mappedFile.setFlushedPosition(pos); +- this.maxPhysicOffset = offset + size; ++ this.setMaxPhysicOffset(offset + size); + if (isExtAddr(tagsCode)) { + maxExtAddr = tagsCode; + } +@@ -881,8 +881,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, + final long cqOffset) { + +- if (offset + size <= this.maxPhysicOffset) { +- log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset); ++ if (offset + size <= this.getMaxPhysicOffset()) { ++ log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", this.getMaxPhysicOffset(), offset); + return true; + } + +@@ -926,7 +926,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + ); + } + } +- this.maxPhysicOffset = offset + size; ++ this.setMaxPhysicOffset(offset + size); + return mappedFile.appendMessage(this.byteBufferIndex.array()); + } + return false; +@@ -1130,7 +1130,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + @Override + public void destroy() { +- this.maxPhysicOffset = -1; ++ this.setMaxPhysicOffset(-1); + this.minLogicOffset = 0; + this.mappedFileQueue.destroy(); + if (isExtReadEnable()) { +diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +index 0bc70642f..32b90d14f 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +@@ -285,7 +285,7 @@ public class MappedFileQueue implements Swappable { + if (this.mappedFiles.isEmpty()) + return 0; + +- long committed = this.flushedWhere; ++ long committed = this.getFlushedWhere(); + if (committed != 0) { + MappedFile mappedFile = this.getLastMappedFile(0, false); + if (mappedFile != null) { +@@ -442,11 +442,11 @@ public class MappedFileQueue implements Swappable { + } + + public long remainHowManyDataToCommit() { +- return getMaxWrotePosition() - committedWhere; ++ return getMaxWrotePosition() - getCommittedWhere(); + } + + public long remainHowManyDataToFlush() { +- return getMaxOffset() - flushedWhere; ++ return getMaxOffset() - this.getFlushedWhere(); + } + + public void deleteLastMappedFile() { +@@ -616,15 +616,15 @@ public class MappedFileQueue implements Swappable { + + public boolean flush(final int flushLeastPages) { + boolean result = true; +- MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0); ++ MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0); + if (mappedFile != null) { + long tmpTimeStamp = mappedFile.getStoreTimestamp(); + int offset = mappedFile.flush(flushLeastPages); + long where = mappedFile.getFileFromOffset() + offset; +- result = where == this.flushedWhere; +- this.flushedWhere = where; ++ result = where == this.getFlushedWhere(); ++ this.setFlushedWhere(where); + if (0 == flushLeastPages) { +- this.storeTimestamp = tmpTimeStamp; ++ this.setStoreTimestamp(tmpTimeStamp); + } + } + +@@ -633,12 +633,12 @@ public class MappedFileQueue implements Swappable { + + public synchronized boolean commit(final int commitLeastPages) { + boolean result = true; +- MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0); ++ MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0); + if (mappedFile != null) { + int offset = mappedFile.commit(commitLeastPages); + long where = mappedFile.getFileFromOffset() + offset; +- result = where == this.committedWhere; +- this.committedWhere = where; ++ result = where == this.getCommittedWhere(); ++ this.setCommittedWhere(where); + } + + return result; +@@ -763,7 +763,7 @@ public class MappedFileQueue implements Swappable { + mf.destroy(1000 * 3); + } + this.mappedFiles.clear(); +- this.flushedWhere = 0; ++ this.setFlushedWhere(0); + + // delete parent directory + File file = new File(storePath); +@@ -848,6 +848,10 @@ public class MappedFileQueue implements Swappable { + return storeTimestamp; + } + ++ public void setStoreTimestamp(long storeTimestamp) { ++ this.storeTimestamp = storeTimestamp; ++ } ++ + public List getMappedFiles() { + return mappedFiles; + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java +index 8f5af9438..8ff050dfe 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java +@@ -16,7 +16,6 @@ + */ + package org.apache.rocketmq.store; + +- + import java.util.Arrays; + import java.util.HashSet; + import java.util.Set; +@@ -113,8 +112,7 @@ public class MultiPathMappedFileQueue extends MappedFileQueue { + mf.destroy(1000 * 3); + } + this.mappedFiles.clear(); +- this.flushedWhere = 0; +- ++ this.setFlushedWhere(0); + + Set storePathSet = getPaths(); + storePathSet.addAll(getReadonlyPaths()); +-- +2.32.0.windows.2 + + +From 3884f595949462044c5cb3c236199bc1d7ad2341 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= + +Date: Wed, 23 Aug 2023 11:10:30 +0800 +Subject: [PATCH 5/7] [ISSUE #7149] When creating and updating Topic, there + will be problems with permission settings (#7151) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings + +* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings + +* [issue#7249] + +--------- + +Co-authored-by: 十真 +--- + .../main/java/org/apache/rocketmq/broker/BrokerController.java | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index 13f9d002b..e8f943702 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -1733,7 +1733,8 @@ public class BrokerController { + new TopicConfig(topicConfig.getTopicName(), + topicConfig.getReadQueueNums(), + topicConfig.getWriteQueueNums(), +- this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); ++ topicConfig.getPerm() ++ & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); + } else { + registerTopicConfig = new TopicConfig(topicConfig); + } +-- +2.32.0.windows.2 + + +From 017ad110475e8024585327b44f47e5e97aabc63b Mon Sep 17 00:00:00 2001 +From: echooymxq +Date: Wed, 23 Aug 2023 11:11:42 +0800 +Subject: [PATCH 6/7] [ISSUE #7219] Fix Concurrent modify syncStateSet and Mark + synchronizing frequently when shrink. (#7220) + +--- + .../broker/controller/ReplicasManager.java | 29 ++++++++++--------- + .../ha/autoswitch/AutoSwitchHAService.java | 21 ++++++++------ + 2 files changed, 28 insertions(+), 22 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +index abae7cdb0..37c82e434 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +@@ -542,7 +542,7 @@ public class ReplicasManager { + this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId()); + this.tempBrokerMetadata.clear(); + this.brokerControllerId = this.brokerMetadata.getBrokerId(); +- this.haService.setBrokerControllerId(this.brokerControllerId); ++ this.haService.setLocalBrokerId(this.brokerControllerId); + return true; + } catch (Exception e) { + LOGGER.error("fail to create metadata file", e); +@@ -594,7 +594,7 @@ public class ReplicasManager { + if (this.brokerMetadata.isLoaded()) { + this.registerState = RegisterState.CREATE_METADATA_FILE_DONE; + this.brokerControllerId = brokerMetadata.getBrokerId(); +- this.haService.setBrokerControllerId(this.brokerControllerId); ++ this.haService.setLocalBrokerId(this.brokerControllerId); + return; + } + // 2. check if temp metadata exist +@@ -735,23 +735,26 @@ public class ReplicasManager { + if (this.checkSyncStateSetTaskFuture != null) { + this.checkSyncStateSetTaskFuture.cancel(false); + } +- this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(() -> { +- checkSyncStateSetAndDoReport(); +- }, 3 * 1000, this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); ++ this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000, ++ this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); + } + + private void checkSyncStateSetAndDoReport() { +- final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); +- newSyncStateSet.add(this.brokerControllerId); +- synchronized (this) { +- if (this.syncStateSet != null) { +- // Check if syncStateSet changed +- if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { +- return; ++ try { ++ final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); ++ newSyncStateSet.add(this.brokerControllerId); ++ synchronized (this) { ++ if (this.syncStateSet != null) { ++ // Check if syncStateSet changed ++ if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { ++ return; ++ } + } + } ++ doReportSyncStateSetChanged(newSyncStateSet); ++ } catch (Exception e) { ++ LOGGER.error("Check syncStateSet error", e); + } +- doReportSyncStateSetChanged(newSyncStateSet); + } + + private void doReportSyncStateSetChanged(Set newSyncStateSet) { +diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +index 6dc734e0c..d5393fdca 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +@@ -41,6 +41,7 @@ import java.nio.channels.SocketChannel; + import java.util.ArrayList; + import java.util.HashSet; + import java.util.List; ++import java.util.Iterator; + import java.util.Map; + import java.util.Objects; + import java.util.Set; +@@ -73,7 +74,7 @@ public class AutoSwitchHAService extends DefaultHAService { + private EpochFileCache epochCache; + private AutoSwitchHAClient haClient; + +- private Long brokerControllerId = null; ++ private Long localBrokerId = null; + + public AutoSwitchHAService() { + } +@@ -287,9 +288,11 @@ public class AutoSwitchHAService extends DefaultHAService { + + // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable, + // it means that the broker has not connected. +- for (Long slaveBrokerId : newSyncStateSet) { +- if (!this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { +- newSyncStateSet.remove(slaveBrokerId); ++ Iterator iterator = newSyncStateSet.iterator(); ++ while (iterator.hasNext()) { ++ Long slaveBrokerId = iterator.next(); ++ if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { ++ iterator.remove(); + isSyncStateSetChanged = true; + } + } +@@ -419,7 +422,7 @@ public class AutoSwitchHAService extends DefaultHAService { + // To avoid the syncStateSet is not consistent with connectionList. + // Fix issue: https://github.com/apache/rocketmq/issues/6662 + for (Long syncId : currentSyncStateSet) { +- if (!idList.contains(syncId) && this.brokerControllerId != null && !Objects.equals(syncId, this.brokerControllerId)) { ++ if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) { + LOGGER.warn("Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.", syncId); + // Without check and re-compute, return the confirmOffset's value directly. + return this.defaultMessageStore.getConfirmOffsetDirectly(); +@@ -545,12 +548,12 @@ public class AutoSwitchHAService extends DefaultHAService { + return this.epochCache.getAllEntries(); + } + +- public Long getBrokerControllerId() { +- return brokerControllerId; ++ public Long getLocalBrokerId() { ++ return localBrokerId; + } + +- public void setBrokerControllerId(Long brokerControllerId) { +- this.brokerControllerId = brokerControllerId; ++ public void setLocalBrokerId(Long localBrokerId) { ++ this.localBrokerId = localBrokerId; + } + + class AutoSwitchAcceptSocketService extends AcceptSocketService { +-- +2.32.0.windows.2 + + +From 77e8e54b37c3fc3ea0beffc1ace6f5bf20af10d9 Mon Sep 17 00:00:00 2001 +From: lk +Date: Wed, 23 Aug 2023 15:56:39 +0800 +Subject: [PATCH 7/7] [ISSUE #7223] Support batch ack for grpc client in proxy + (#7225) + +--- + .../client/impl/mqclient/MQClientAPIExt.java | 26 +++ + .../rocketmq/proxy/config/ProxyConfig.java | 10 + + .../grpc/v2/consumer/AckMessageActivity.java | 136 ++++++++--- + .../proxy/processor/AbstractProcessor.java | 4 +- + .../proxy/processor/BatchAckResult.java | 53 +++++ + .../proxy/processor/ConsumerProcessor.java | 64 +++++ + .../processor/DefaultMessagingProcessor.java | 7 + + .../proxy/processor/MessagingProcessor.java | 18 ++ + .../message/ClusterMessageService.java | 16 +- + .../service/message/LocalMessageService.java | 58 +++++ + .../proxy/service/message/MessageService.java | 8 + + .../service/message/ReceiptHandleMessage.java | 39 ++++ + .../v2/consumer/AckMessageActivityTest.java | 221 +++++++++++++++--- + .../proxy/processor/BaseProcessorTest.java | 18 +- + .../processor/ConsumerProcessorTest.java | 115 +++++++++ + .../service/mqclient/MQClientAPIExtTest.java | 12 + + 16 files changed, 728 insertions(+), 77 deletions(-) + create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java + create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java + +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +index fb8f8d11f..d7c8ef8d9 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +@@ -306,6 +306,32 @@ public class MQClientAPIExt extends MQClientAPIImpl { + return future; + } + ++ public CompletableFuture batchAckMessageAsync( ++ String brokerAddr, ++ String topic, ++ String consumerGroup, ++ List extraInfoList, ++ long timeoutMillis ++ ) { ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++ this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() { ++ @Override ++ public void onSuccess(AckResult ackResult) { ++ future.complete(ackResult); ++ } ++ ++ @Override ++ public void onException(Throwable t) { ++ future.completeExceptionally(t); ++ } ++ }, topic, consumerGroup, extraInfoList); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++ return future; ++ } ++ + public CompletableFuture changeInvisibleTimeAsync( + String brokerAddr, + String brokerName, +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +index 39caaa0d9..76a243919 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +@@ -250,6 +250,8 @@ public class ProxyConfig implements ConfigFile { + private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000; + private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000; + ++ private boolean enableBatchAck = false; ++ + @Override + public void initData() { + parseDelayLevel(); +@@ -1379,4 +1381,12 @@ public class ProxyConfig implements ConfigFile { + public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) { + this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; + } ++ ++ public boolean isEnableBatchAck() { ++ return enableBatchAck; ++ } ++ ++ public void setEnableBatchAck(boolean enableBatchAck) { ++ this.enableBatchAck = enableBatchAck; ++ } + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java +index 9a3a77201..97c716c8f 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java +@@ -31,12 +31,15 @@ import org.apache.rocketmq.client.consumer.AckStatus; + import org.apache.rocketmq.common.consumer.ReceiptHandle; + import org.apache.rocketmq.proxy.common.MessageReceiptHandle; + import org.apache.rocketmq.proxy.common.ProxyContext; ++import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; + import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; + import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; + import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; + import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; ++import org.apache.rocketmq.proxy.processor.BatchAckResult; + import org.apache.rocketmq.proxy.processor.MessagingProcessor; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + + public class AckMessageActivity extends AbstractMessingActivity { + +@@ -50,60 +53,98 @@ public class AckMessageActivity extends AbstractMessingActivity { + + try { + validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); +- +- CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; +- for (int i = 0; i < request.getEntriesCount(); i++) { +- futures[i] = processAckMessage(ctx, request, request.getEntries(i)); ++ String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); ++ String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()); ++ if (ConfigurationManager.getProxyConfig().isEnableBatchAck()) { ++ future = ackMessageInBatch(ctx, group, topic, request); ++ } else { ++ future = ackMessageOneByOne(ctx, group, topic, request); + } +- CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +- if (throwable != null) { +- future.completeExceptionally(throwable); +- return; +- } ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++ return future; ++ } ++ ++ protected CompletableFuture ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) { ++ List handleMessageList = new ArrayList<>(request.getEntriesCount()); + ++ for (AckMessageEntry ackMessageEntry : request.getEntriesList()) { ++ String handleString = getHandleString(ctx, group, request, ackMessageEntry); ++ handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId())); ++ } ++ return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic) ++ .thenApply(batchAckResultList -> { ++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder(); + Set responseCodes = new HashSet<>(); +- List entryList = new ArrayList<>(); +- for (CompletableFuture entryFuture : futures) { +- AckMessageResultEntry entryResult = entryFuture.join(); +- responseCodes.add(entryResult.getStatus().getCode()); +- entryList.add(entryResult); +- } +- AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() +- .addAllEntries(entryList); +- if (responseCodes.size() > 1) { +- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); +- } else if (responseCodes.size() == 1) { +- Code code = responseCodes.stream().findAny().get(); +- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); +- } else { +- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); ++ for (BatchAckResult batchAckResult : batchAckResultList) { ++ AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult); ++ responseBuilder.addEntries(entry); ++ responseCodes.add(entry.getStatus().getCode()); + } +- future.complete(responseBuilder.build()); ++ setAckResponseStatus(responseBuilder, responseCodes); ++ return responseBuilder.build(); + }); +- } catch (Throwable t) { +- future.completeExceptionally(t); ++ } ++ ++ protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) { ++ ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage(); ++ AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder() ++ .setMessageId(handleMessage.getMessageId()) ++ .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle()); ++ if (batchAckResult.getProxyException() != null) { ++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException())); ++ } else { ++ AckResult ackResult = batchAckResult.getAckResult(); ++ if (AckStatus.OK.equals(ackResult.getStatus())) { ++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); ++ } else { ++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")); ++ } + } +- return future; ++ return resultBuilder.build(); + } + +- protected CompletableFuture processAckMessage(ProxyContext ctx, AckMessageRequest request, ++ protected CompletableFuture ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) { ++ CompletableFuture resultFuture = new CompletableFuture<>(); ++ CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; ++ for (int i = 0; i < request.getEntriesCount(); i++) { ++ futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i)); ++ } ++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { ++ if (throwable != null) { ++ resultFuture.completeExceptionally(throwable); ++ return; ++ } ++ ++ Set responseCodes = new HashSet<>(); ++ List entryList = new ArrayList<>(); ++ for (CompletableFuture entryFuture : futures) { ++ AckMessageResultEntry entryResult = entryFuture.join(); ++ responseCodes.add(entryResult.getStatus().getCode()); ++ entryList.add(entryResult); ++ } ++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() ++ .addAllEntries(entryList); ++ setAckResponseStatus(responseBuilder, responseCodes); ++ resultFuture.complete(responseBuilder.build()); ++ }); ++ return resultFuture; ++ } ++ ++ protected CompletableFuture processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request, + AckMessageEntry ackMessageEntry) { + CompletableFuture future = new CompletableFuture<>(); + + try { +- String handleString = ackMessageEntry.getReceiptHandle(); +- +- String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); +- MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); +- if (messageReceiptHandle != null) { +- handleString = messageReceiptHandle.getReceiptHandleStr(); +- } ++ String handleString = this.getHandleString(ctx, group, request, ackMessageEntry); + CompletableFuture ackResultFuture = this.messagingProcessor.ackMessage( + ctx, + ReceiptHandle.decode(handleString), + ackMessageEntry.getMessageId(), + group, +- GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); ++ topic ++ ); + ackResultFuture.thenAccept(result -> { + future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result)); + }).exceptionally(t -> { +@@ -139,4 +180,25 @@ public class AckMessageActivity extends AbstractMessingActivity { + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")) + .build(); + } ++ ++ protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set responseCodes) { ++ if (responseCodes.size() > 1) { ++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); ++ } else if (responseCodes.size() == 1) { ++ Code code = responseCodes.stream().findAny().get(); ++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); ++ } else { ++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); ++ } ++ } ++ ++ protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) { ++ String handleString = ackMessageEntry.getReceiptHandle(); ++ ++ MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); ++ if (messageReceiptHandle != null) { ++ handleString = messageReceiptHandle.getReceiptHandleStr(); ++ } ++ return handleString; ++ } + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java +index b61c3df9e..c63212c23 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java +@@ -27,6 +27,8 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { + protected MessagingProcessor messagingProcessor; + protected ServiceManager serviceManager; + ++ protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); ++ + public AbstractProcessor(MessagingProcessor messagingProcessor, + ServiceManager serviceManager) { + this.messagingProcessor = messagingProcessor; +@@ -35,7 +37,7 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { + + protected void validateReceiptHandle(ReceiptHandle handle) { + if (handle.isExpired()) { +- throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); ++ throw EXPIRED_HANDLE_PROXY_EXCEPTION; + } + } + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java +new file mode 100644 +index 000000000..dfb9c9b9e +--- /dev/null ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java +@@ -0,0 +1,53 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.proxy.processor; ++ ++import org.apache.rocketmq.client.consumer.AckResult; ++import org.apache.rocketmq.proxy.common.ProxyException; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ ++public class BatchAckResult { ++ ++ private final ReceiptHandleMessage receiptHandleMessage; ++ private AckResult ackResult; ++ private ProxyException proxyException; ++ ++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, ++ AckResult ackResult) { ++ this.receiptHandleMessage = receiptHandleMessage; ++ this.ackResult = ackResult; ++ } ++ ++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, ++ ProxyException proxyException) { ++ this.receiptHandleMessage = receiptHandleMessage; ++ this.proxyException = proxyException; ++ } ++ ++ public ReceiptHandleMessage getReceiptHandleMessage() { ++ return receiptHandleMessage; ++ } ++ ++ public AckResult getAckResult() { ++ return ackResult; ++ } ++ ++ public ProxyException getProxyException() { ++ return proxyException; ++ } ++} +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +index 656a6339d..f3522b374 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +@@ -48,6 +48,7 @@ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; + import org.apache.rocketmq.proxy.common.utils.FutureUtils; + import org.apache.rocketmq.proxy.common.utils.ProxyUtils; + import org.apache.rocketmq.proxy.service.ServiceManager; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; + import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; + import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; +@@ -241,6 +242,69 @@ public class ConsumerProcessor extends AbstractProcessor { + return FutureUtils.addExecutor(future, this.executor); + } + ++ public CompletableFuture> batchAckMessage( ++ ProxyContext ctx, ++ List handleMessageList, ++ String consumerGroup, ++ String topic, ++ long timeoutMillis ++ ) { ++ CompletableFuture> future = new CompletableFuture<>(); ++ try { ++ List batchAckResultList = new ArrayList<>(handleMessageList.size()); ++ Map> brokerHandleListMap = new HashMap<>(); ++ ++ for (ReceiptHandleMessage handleMessage : handleMessageList) { ++ if (handleMessage.getReceiptHandle().isExpired()) { ++ batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION)); ++ continue; ++ } ++ List brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>()); ++ brokerHandleList.add(handleMessage); ++ } ++ ++ if (brokerHandleListMap.isEmpty()) { ++ return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor); ++ } ++ Set>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet(); ++ CompletableFuture>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()]; ++ int futureIndex = 0; ++ for (Map.Entry> entry : brokerHandleListMapEntrySet) { ++ futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis); ++ } ++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { ++ if (throwable != null) { ++ future.completeExceptionally(throwable); ++ } ++ for (CompletableFuture> resultFuture : futures) { ++ batchAckResultList.addAll(resultFuture.join()); ++ } ++ future.complete(batchAckResultList); ++ }); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++ return FutureUtils.addExecutor(future, this.executor); ++ } ++ ++ protected CompletableFuture> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List handleMessageList, long timeoutMillis) { ++ return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis) ++ .thenApply(result -> { ++ List results = new ArrayList<>(); ++ for (ReceiptHandleMessage handleMessage : handleMessageList) { ++ results.add(new BatchAckResult(handleMessage, result)); ++ } ++ return results; ++ }) ++ .exceptionally(throwable -> { ++ List results = new ArrayList<>(); ++ for (ReceiptHandleMessage handleMessage : handleMessageList) { ++ results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable))); ++ } ++ return results; ++ }); ++ } ++ + public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, + String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +index 188cb7b9b..ba150051b 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +@@ -46,6 +46,7 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.config.ProxyConfig; + import org.apache.rocketmq.proxy.service.ServiceManager; + import org.apache.rocketmq.proxy.service.ServiceManagerFactory; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.apache.rocketmq.proxy.service.metadata.MetadataService; + import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; + import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; +@@ -183,6 +184,12 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen + return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis); + } + ++ @Override ++ public CompletableFuture> batchAckMessage(ProxyContext ctx, ++ List handleMessageList, String consumerGroup, String topic, long timeoutMillis) { ++ return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis); ++ } ++ + @Override + public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, + String groupName, String topicName, long invisibleTime, long timeoutMillis) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +index d86be0bd8..2ae7418ba 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +@@ -37,6 +37,7 @@ import org.apache.rocketmq.proxy.common.Address; + import org.apache.rocketmq.proxy.common.MessageReceiptHandle; + import org.apache.rocketmq.proxy.common.ProxyContext; + import org.apache.rocketmq.common.utils.StartAndShutdown; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.apache.rocketmq.proxy.service.metadata.MetadataService; + import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; + import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; +@@ -155,6 +156,23 @@ public interface MessagingProcessor extends StartAndShutdown { + long timeoutMillis + ); + ++ default CompletableFuture> batchAckMessage( ++ ProxyContext ctx, ++ List handleMessageList, ++ String consumerGroup, ++ String topic ++ ) { ++ return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS); ++ } ++ ++ CompletableFuture> batchAckMessage( ++ ProxyContext ctx, ++ List handleMessageList, ++ String consumerGroup, ++ String topic, ++ long timeoutMillis ++ ); ++ + default CompletableFuture changeInvisibleTime( + ProxyContext ctx, + ReceiptHandle handle, +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +index 9f163f1b9..70b72deae 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +@@ -20,9 +20,11 @@ import com.google.common.collect.Lists; + import java.util.List; + import java.util.Set; + import java.util.concurrent.CompletableFuture; ++import java.util.stream.Collectors; + import org.apache.rocketmq.client.consumer.AckResult; + import org.apache.rocketmq.client.consumer.PopResult; + import org.apache.rocketmq.client.consumer.PullResult; ++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; + import org.apache.rocketmq.client.producer.SendResult; + import org.apache.rocketmq.common.consumer.ReceiptHandle; + import org.apache.rocketmq.common.message.Message; +@@ -31,7 +33,6 @@ import org.apache.rocketmq.proxy.common.ProxyContext; + import org.apache.rocketmq.proxy.common.ProxyException; + import org.apache.rocketmq.proxy.common.ProxyExceptionCode; + import org.apache.rocketmq.proxy.common.utils.FutureUtils; +-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; + import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; + import org.apache.rocketmq.proxy.service.route.TopicRouteService; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; +@@ -137,6 +138,19 @@ public class ClusterMessageService implements MessageService { + ); + } + ++ @Override ++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, String consumerGroup, ++ String topic, long timeoutMillis) { ++ List extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList()); ++ return this.mqClientAPIFactory.getClient().batchAckMessageAsync( ++ this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()), ++ topic, ++ consumerGroup, ++ extraInfoList, ++ timeoutMillis ++ ); ++ } ++ + @Override + public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + PullMessageRequestHeader requestHeader, long timeoutMillis) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +index eb2c4d9ee..ca7dcc9eb 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.service.message; + import io.netty.channel.ChannelHandlerContext; + import java.nio.ByteBuffer; + import java.util.ArrayList; ++import java.util.BitSet; + import java.util.Collections; + import java.util.HashMap; + import java.util.List; +@@ -54,6 +55,8 @@ import org.apache.rocketmq.remoting.RPCHook; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; + import org.apache.rocketmq.remoting.protocol.RequestCode; + import org.apache.rocketmq.remoting.protocol.ResponseCode; ++import org.apache.rocketmq.remoting.protocol.body.BatchAck; ++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; + import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; + import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; + import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; +@@ -364,6 +367,61 @@ public class LocalMessageService implements MessageService { + }); + } + ++ @Override ++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, ++ String consumerGroup, String topic, long timeoutMillis) { ++ SimpleChannel channel = channelManager.createChannel(ctx); ++ ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); ++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); ++ ++ Map batchAckMap = new HashMap<>(); ++ for (ReceiptHandleMessage receiptHandleMessage : handleList) { ++ String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle(); ++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo); ++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + ++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" + ++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + ++ ExtraInfoUtil.getPopTime(extraInfoData); ++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { ++ BatchAck newBatchAck = new BatchAck(); ++ newBatchAck.setConsumerGroup(consumerGroup); ++ newBatchAck.setTopic(topic); ++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); ++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); ++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); ++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); ++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); ++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); ++ newBatchAck.setBitSet(new BitSet()); ++ return newBatchAck; ++ }); ++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); ++ } ++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); ++ requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName()); ++ requestBody.setAcks(new ArrayList<>(batchAckMap.values())); ++ ++ command.setBody(requestBody.encode()); ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++ RemotingCommand response = brokerController.getAckMessageProcessor() ++ .processRequest(channelHandlerContext, command); ++ future.complete(response); ++ } catch (Exception e) { ++ log.error("Fail to process batchAckMessage command", e); ++ future.completeExceptionally(e); ++ } ++ return future.thenApply(r -> { ++ AckResult ackResult = new AckResult(); ++ if (ResponseCode.SUCCESS == r.getCode()) { ++ ackResult.setStatus(AckStatus.OK); ++ } else { ++ ackResult.setStatus(AckStatus.NO_EXIST); ++ } ++ return ackResult; ++ }); ++ } ++ + @Override + public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + PullMessageRequestHeader requestHeader, long timeoutMillis) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +index 15da17154..58a835adb 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +@@ -91,6 +91,14 @@ public interface MessageService { + long timeoutMillis + ); + ++ CompletableFuture batchAckMessage( ++ ProxyContext ctx, ++ List handleList, ++ String consumerGroup, ++ String topic, ++ long timeoutMillis ++ ); ++ + CompletableFuture pullMessage( + ProxyContext ctx, + AddressableMessageQueue messageQueue, +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java +new file mode 100644 +index 000000000..ae63fed49 +--- /dev/null ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java +@@ -0,0 +1,39 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.proxy.service.message; ++ ++import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ ++public class ReceiptHandleMessage { ++ ++ private final ReceiptHandle receiptHandle; ++ private final String messageId; ++ ++ public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) { ++ this.receiptHandle = receiptHandle; ++ this.messageId = messageId; ++ } ++ ++ public ReceiptHandle getReceiptHandle() { ++ return receiptHandle; ++ } ++ ++ public String getMessageId() { ++ return messageId; ++ } ++} +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java +index 49fdfc6a8..3c4746105 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java +@@ -20,21 +20,32 @@ package org.apache.rocketmq.proxy.grpc.v2.consumer; + import apache.rocketmq.v2.AckMessageEntry; + import apache.rocketmq.v2.AckMessageRequest; + import apache.rocketmq.v2.AckMessageResponse; ++import apache.rocketmq.v2.AckMessageResultEntry; + import apache.rocketmq.v2.Code; + import apache.rocketmq.v2.Resource; ++import java.util.ArrayList; ++import java.util.HashMap; ++import java.util.List; ++import java.util.Map; + import java.util.concurrent.CompletableFuture; + import org.apache.rocketmq.client.consumer.AckResult; + import org.apache.rocketmq.client.consumer.AckStatus; + import org.apache.rocketmq.proxy.common.ProxyException; + import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; ++import org.apache.rocketmq.proxy.processor.BatchAckResult; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.junit.Before; + import org.junit.Test; ++import org.mockito.stubbing.Answer; + + import static org.junit.Assert.assertEquals; + import static org.mockito.ArgumentMatchers.any; ++import static org.mockito.ArgumentMatchers.anyList; + import static org.mockito.ArgumentMatchers.anyString; + import static org.mockito.ArgumentMatchers.eq; ++import static org.mockito.Mockito.doAnswer; + import static org.mockito.Mockito.when; + + public class AckMessageActivityTest extends BaseActivityTest { +@@ -52,43 +63,197 @@ public class AckMessageActivityTest extends BaseActivityTest { + + @Test + public void testAckMessage() throws Throwable { +- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString())) ++ ConfigurationManager.getProxyConfig().setEnableBatchAck(false); ++ ++ String msg1 = "msg1"; ++ String msg2 = "msg2"; ++ String msg3 = "msg3"; ++ ++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString())) + .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired")); + + AckResult msg2AckResult = new AckResult(); + msg2AckResult.setStatus(AckStatus.OK); +- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString())) ++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString())) + .thenReturn(CompletableFuture.completedFuture(msg2AckResult)); + + AckResult msg3AckResult = new AckResult(); + msg3AckResult.setStatus(AckStatus.NO_EXIST); +- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString())) ++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString())) + .thenReturn(CompletableFuture.completedFuture(msg3AckResult)); + +- AckMessageResponse response = this.ackMessageActivity.ackMessage( +- createContext(), +- AckMessageRequest.newBuilder() +- .setTopic(Resource.newBuilder().setName(TOPIC).build()) +- .setGroup(Resource.newBuilder().setName(GROUP).build()) +- .addEntries(AckMessageEntry.newBuilder() +- .setMessageId("msg1") +- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +- .build()) +- .addEntries(AckMessageEntry.newBuilder() +- .setMessageId("msg2") +- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +- .build()) +- .addEntries(AckMessageEntry.newBuilder() +- .setMessageId("msg3") +- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +- .build()) +- .build() +- ).get(); +- +- assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +- assertEquals(3, response.getEntriesCount()); +- assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); +- assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); +- assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg1) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg2) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.OK, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg3) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg1) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++ .build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg2) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg3) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ ++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); ++ assertEquals(3, response.getEntriesCount()); ++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); ++ assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); ++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); ++ } ++ } ++ ++ @Test ++ public void testAckMessageInBatch() throws Throwable { ++ ConfigurationManager.getProxyConfig().setEnableBatchAck(true); ++ ++ String successMessageId = "msg1"; ++ String notOkMessageId = "msg2"; ++ String exceptionMessageId = "msg3"; ++ ++ doAnswer((Answer>>) invocation -> { ++ List receiptHandleMessageList = invocation.getArgument(1, List.class); ++ List batchAckResultList = new ArrayList<>(); ++ for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) { ++ BatchAckResult batchAckResult; ++ if (receiptHandleMessage.getMessageId().equals(successMessageId)) { ++ AckResult ackResult = new AckResult(); ++ ackResult.setStatus(AckStatus.OK); ++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); ++ } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) { ++ AckResult ackResult = new AckResult(); ++ ackResult.setStatus(AckStatus.NO_EXIST); ++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); ++ } else { ++ batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "")); ++ } ++ batchAckResultList.add(batchAckResult); ++ } ++ return CompletableFuture.completedFuture(batchAckResultList); ++ }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString()); ++ ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(successMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.OK, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(notOkMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(exceptionMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(successMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(notOkMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(exceptionMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ ++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); ++ assertEquals(3, response.getEntriesCount()); ++ Map msgCode = new HashMap<>(); ++ for (AckMessageResultEntry entry : response.getEntriesList()) { ++ msgCode.put(entry.getMessageId(), entry.getStatus().getCode()); ++ } ++ assertEquals(Code.OK, msgCode.get(successMessageId)); ++ assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId)); ++ assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId)); ++ } + } + } +\ No newline at end of file +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java +index 5c1ea9627..072630e39 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java +@@ -66,14 +66,6 @@ public class BaseProcessorTest extends InitConfigTest { + protected ProxyRelayService proxyRelayService; + @Mock + protected MetadataService metadataService; +- @Mock +- protected ProducerProcessor producerProcessor; +- @Mock +- protected ConsumerProcessor consumerProcessor; +- @Mock +- protected TransactionProcessor transactionProcessor; +- @Mock +- protected ClientProcessor clientProcessor; + + public void before() throws Throwable { + super.before(); +@@ -92,6 +84,13 @@ public class BaseProcessorTest extends InitConfigTest { + } + + protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) { ++ return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(), ++ RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), ++ RANDOM.nextInt(Integer.MAX_VALUE), "mockBroker"); ++ } ++ ++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime, ++ long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) { + MessageExt messageExt = new MessageExt(); + messageExt.setTopic(topic); + messageExt.setTags(tags); +@@ -100,8 +99,7 @@ public class BaseProcessorTest extends InitConfigTest { + messageExt.setMsgId(MessageClientIDSetter.createUniqID()); + messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE)); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, +- ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime, +- RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE))); ++ ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset)); + return messageExt; + } + +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java +index 717e86fc0..db268a06e 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java +@@ -20,8 +20,11 @@ package org.apache.rocketmq.proxy.processor; + import com.google.common.collect.Sets; + import java.time.Duration; + import java.util.ArrayList; ++import java.util.Collections; ++import java.util.HashMap; + import java.util.HashSet; + import java.util.List; ++import java.util.Map; + import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.Executors; +@@ -39,7 +42,10 @@ import org.apache.rocketmq.common.message.MessageClientIDSetter; + import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.proxy.common.ProxyContext; ++import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++import org.apache.rocketmq.proxy.common.utils.FutureUtils; + import org.apache.rocketmq.proxy.common.utils.ProxyUtils; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; + import org.apache.rocketmq.proxy.service.route.MessageQueueView; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; +@@ -50,16 +56,22 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; + import org.junit.Before; + import org.junit.Test; + import org.mockito.ArgumentCaptor; ++import org.mockito.stubbing.Answer; + + import static org.assertj.core.api.Assertions.assertThat; + import static org.junit.Assert.assertEquals; + import static org.junit.Assert.assertNotNull; ++import static org.junit.Assert.assertNull; + import static org.junit.Assert.assertSame; + import static org.mockito.ArgumentMatchers.any; ++import static org.mockito.ArgumentMatchers.anyList; + import static org.mockito.ArgumentMatchers.anyLong; + import static org.mockito.ArgumentMatchers.anyString; + import static org.mockito.ArgumentMatchers.eq; ++import static org.mockito.Mockito.doAnswer; + import static org.mockito.Mockito.mock; ++import static org.mockito.Mockito.never; ++import static org.mockito.Mockito.verify; + import static org.mockito.Mockito.when; + + public class ConsumerProcessorTest extends BaseProcessorTest { +@@ -162,6 +174,109 @@ public class ConsumerProcessorTest extends BaseProcessorTest { + assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo()); + } + ++ @Test ++ public void testBatchAckExpireMessage() throws Throwable { ++ String brokerName1 = "brokerName1"; ++ ++ List receiptHandleMessageList = new ArrayList<>(); ++ for (int i = 0; i < 3; i++) { ++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, ++ 0, 0, 0, i, brokerName1); ++ ReceiptHandle expireHandle = create(expireMessage); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); ++ } ++ ++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); ++ ++ verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); ++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); ++ for (BatchAckResult batchAckResult : batchAckResultList) { ++ assertNull(batchAckResult.getAckResult()); ++ assertNotNull(batchAckResult.getProxyException()); ++ assertNotNull(batchAckResult.getReceiptHandleMessage()); ++ } ++ ++ } ++ ++ @Test ++ public void testBatchAckMessage() throws Throwable { ++ String brokerName1 = "brokerName1"; ++ String brokerName2 = "brokerName2"; ++ String errThrowBrokerName = "errThrowBrokerName"; ++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, ++ 0, 0, 0, 0, brokerName1); ++ ReceiptHandle expireHandle = create(expireMessage); ++ ++ List receiptHandleMessageList = new ArrayList<>(); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); ++ List broker1Msg = new ArrayList<>(); ++ List broker2Msg = new ArrayList<>(); ++ ++ long now = System.currentTimeMillis(); ++ int msgNum = 3; ++ for (int i = 0; i < msgNum; i++) { ++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, ++ 0, 0, 0, i + 1, brokerName1); ++ ReceiptHandle brokerHandle = create(brokerMessage); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); ++ broker1Msg.add(brokerMessage.getMsgId()); ++ } ++ for (int i = 0; i < msgNum; i++) { ++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, ++ 0, 0, 0, i + 1, brokerName2); ++ ReceiptHandle brokerHandle = create(brokerMessage); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); ++ broker2Msg.add(brokerMessage.getMsgId()); ++ } ++ ++ // for this message, will throw exception in batchAckMessage ++ MessageExt errThrowMessage = createMessageExt(TOPIC, "", 0, 3000, now, ++ 0, 0, 0, 0, errThrowBrokerName); ++ ReceiptHandle errThrowHandle = create(errThrowMessage); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId())); ++ ++ Collections.shuffle(receiptHandleMessageList); ++ ++ doAnswer((Answer>) invocation -> { ++ List handleMessageList = invocation.getArgument(1, List.class); ++ AckResult ackResult = new AckResult(); ++ String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName(); ++ if (brokerName.equals(brokerName1)) { ++ ackResult.setStatus(AckStatus.OK); ++ } else if (brokerName.equals(brokerName2)) { ++ ackResult.setStatus(AckStatus.NO_EXIST); ++ } else { ++ return FutureUtils.completeExceptionally(new RuntimeException()); ++ } ++ ++ return CompletableFuture.completedFuture(ackResult); ++ }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); ++ ++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); ++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); ++ ++ // check ackResult for each msg ++ Map msgBatchAckResult = new HashMap<>(); ++ for (BatchAckResult batchAckResult : batchAckResultList) { ++ msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult); ++ } ++ for (String msgId : broker1Msg) { ++ assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus()); ++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); ++ } ++ for (String msgId : broker2Msg) { ++ assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus()); ++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); ++ } ++ assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException()); ++ assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode()); ++ assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult()); ++ ++ assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException()); ++ assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode()); ++ assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult()); ++ } ++ + @Test + public void testChangeInvisibleTime() throws Throwable { + ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000)); +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +index 77a119a29..3f3a4ae40 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +@@ -220,6 +220,18 @@ public class MQClientAPIExtTest { + assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get()); + } + ++ @Test ++ public void testBatchAckMessageAsync() throws Exception { ++ AckResult ackResult = new AckResult(); ++ doAnswer((Answer) mock -> { ++ AckCallback ackCallback = mock.getArgument(2); ++ ackCallback.onSuccess(ackResult); ++ return null; ++ }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any()); ++ ++ assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get()); ++ } ++ + @Test + public void testChangeInvisibleTimeAsync() throws Exception { + AckResult ackResult = new AckResult(); +-- +2.32.0.windows.2 + diff --git a/patch012-backport-enhance-rockdbconfigtojson.patch b/patch012-backport-enhance-rockdbconfigtojson.patch new file mode 100644 index 000000000..cf6128b24 --- /dev/null +++ b/patch012-backport-enhance-rockdbconfigtojson.patch @@ -0,0 +1,2920 @@ +From fec141481496c53a0db398367006c34264662d18 Mon Sep 17 00:00:00 2001 +From: yx9o +Date: Wed, 23 Aug 2023 08:22:34 +0800 +Subject: [PATCH 1/8] [ISSUE #7166] Optimize the display format of admin + (#7210) + +--- + .../java/org/apache/rocketmq/tools/command/MQAdminStartup.java | 2 +- + .../command/acl/ClusterAclConfigVersionListSubCommand.java | 2 +- + .../tools/command/acl/DeleteAccessConfigSubCommand.java | 2 +- + .../rocketmq/tools/command/acl/GetAccessConfigSubCommand.java | 2 +- + .../tools/command/acl/UpdateAccessConfigSubCommand.java | 2 +- + .../tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java | 2 +- + .../tools/command/broker/BrokerConsumeStatsSubCommad.java | 2 +- + .../rocketmq/tools/command/broker/BrokerStatusSubCommand.java | 2 +- + .../tools/command/broker/CommitLogSetReadAheadSubCommand.java | 2 +- + .../tools/command/broker/DeleteExpiredCommitLogSubCommand.java | 2 +- + .../rocketmq/tools/command/broker/GetBrokerConfigCommand.java | 2 +- + .../rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java | 2 +- + .../tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java | 2 +- + .../broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java | 2 +- + .../tools/command/broker/ResetMasterFlushOffsetSubCommand.java | 2 +- + .../tools/command/broker/UpdateBrokerConfigSubCommand.java | 2 +- + .../broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java | 2 +- + .../rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java | 2 +- + .../rocketmq/tools/command/cluster/ClusterListSubCommand.java | 2 +- + .../tools/command/connection/ConsumerConnectionSubCommand.java | 2 +- + .../tools/command/connection/ProducerConnectionSubCommand.java | 2 +- + .../tools/command/consumer/ConsumerStatusSubCommand.java | 2 +- + .../tools/command/consumer/GetConsumerConfigSubCommand.java | 2 +- + .../tools/command/consumer/StartMonitoringSubCommand.java | 2 +- + .../tools/command/consumer/UpdateSubGroupSubCommand.java | 2 +- + .../rocketmq/tools/command/container/AddBrokerSubCommand.java | 2 +- + .../tools/command/container/RemoveBrokerSubCommand.java | 2 +- + .../command/controller/CleanControllerBrokerMetaSubCommand.java | 2 +- + .../command/controller/GetControllerMetaDataSubCommand.java | 2 +- + .../tools/command/controller/ReElectMasterSubCommand.java | 2 +- + .../rocketmq/tools/command/export/ExportConfigsCommand.java | 2 +- + .../rocketmq/tools/command/export/ExportMetadataCommand.java | 2 +- + .../rocketmq/tools/command/export/ExportMetricsCommand.java | 2 +- + .../rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java | 2 +- + .../apache/rocketmq/tools/command/ha/HAStatusSubCommand.java | 2 +- + .../rocketmq/tools/command/message/CheckMsgSendRTCommand.java | 2 +- + .../rocketmq/tools/command/message/ConsumeMessageCommand.java | 2 +- + .../tools/command/message/DumpCompactionLogCommand.java | 2 +- + .../tools/command/message/PrintMessageByQueueCommand.java | 2 +- + .../rocketmq/tools/command/message/PrintMessageSubCommand.java | 2 +- + .../rocketmq/tools/command/message/QueryMsgByIdSubCommand.java | 2 +- + .../rocketmq/tools/command/message/QueryMsgByKeySubCommand.java | 2 +- + .../tools/command/message/QueryMsgByOffsetSubCommand.java | 2 +- + .../tools/command/message/QueryMsgByUniqueKeySubCommand.java | 2 +- + .../tools/command/message/QueryMsgTraceByIdSubCommand.java | 2 +- + .../rocketmq/tools/command/message/SendMessageCommand.java | 2 +- + .../rocketmq/tools/command/namesrv/AddWritePermSubCommand.java | 2 +- + .../rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java | 2 +- + .../tools/command/offset/SkipAccumulationSubCommand.java | 2 +- + .../apache/rocketmq/tools/command/stats/StatsAllSubCommand.java | 2 +- + .../rocketmq/tools/command/topic/AllocateMQSubCommand.java | 2 +- + .../rocketmq/tools/command/topic/TopicClusterSubCommand.java | 2 +- + .../rocketmq/tools/command/topic/TopicListSubCommand.java | 2 +- + .../rocketmq/tools/command/topic/TopicRouteSubCommand.java | 2 +- + .../rocketmq/tools/command/topic/TopicStatusSubCommand.java | 2 +- + .../rocketmq/tools/command/topic/UpdateOrderConfCommand.java | 2 +- + .../tools/command/topic/UpdateStaticTopicSubCommand.java | 2 +- + .../rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java | 2 +- + .../rocketmq/tools/command/topic/UpdateTopicSubCommand.java | 2 +- + 59 files changed, 59 insertions(+), 59 deletions(-) + +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +index 0c2618e91..890125ca0 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +@@ -278,7 +278,7 @@ public class MQAdminStartup { + System.out.printf("The most commonly used mqadmin commands are:%n"); + + for (SubCommand cmd : SUB_COMMANDS) { +- System.out.printf(" %-25s %s%n", cmd.commandName(), cmd.commandDesc()); ++ System.out.printf(" %-35s %s%n", cmd.commandName(), cmd.commandDesc()); + } + + System.out.printf("%nSee 'mqadmin help ' for more information on a specific command.%n"); +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java +index f8a00b1e0..26ed028fb 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java +@@ -47,7 +47,7 @@ public class ClusterAclConfigVersionListSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "List all of acl config version information in cluster"; ++ return "List all of acl config version information in cluster."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java +index fd3a92fff..a7f3d295a 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java +@@ -42,7 +42,7 @@ public class DeleteAccessConfigSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Delete Acl Config Account in broker"; ++ return "Delete Acl Config Account in broker."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java +index 25844d6a1..f1c9a1496 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java +@@ -49,7 +49,7 @@ public class GetAccessConfigSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "List all of acl config information in cluster"; ++ return "List all of acl config information in cluster."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java +index 3be40daa1..d8a06f92d 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java +@@ -40,7 +40,7 @@ public class UpdateAccessConfigSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Update acl config yaml file in broker"; ++ return "Update acl config yaml file in broker."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java +index ff662b506..9dacf1fae 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java +@@ -37,7 +37,7 @@ public class UpdateGlobalWhiteAddrSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Update global white address for acl Config File in broker"; ++ return "Update global white address for acl Config File in broker."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java +index 3f2f90673..7658a2139 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java +@@ -61,7 +61,7 @@ public class BrokerConsumeStatsSubCommad implements SubCommand { + + @Override + public String commandDesc() { +- return "Fetch broker consume stats data"; ++ return "Fetch broker consume stats data."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java +index 830ff3425..ce934f547 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java +@@ -44,7 +44,7 @@ public class BrokerStatusSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Fetch broker runtime status data"; ++ return "Fetch broker runtime status data."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java +index b00c7f5f5..4fdabfdf8 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java +@@ -44,7 +44,7 @@ public class CommitLogSetReadAheadSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "set read ahead mode for all commitlog files"; ++ return "Set read ahead mode for all commitlog files."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java +index a4b2a51ad..142bb7b3c 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java +@@ -37,7 +37,7 @@ public class DeleteExpiredCommitLogSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Delete expired CommitLog files"; ++ return "Delete expired CommitLog files."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java +index 5d86c10e4..c4762a296 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java +@@ -45,7 +45,7 @@ public class GetBrokerConfigCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Get broker config by cluster or special broker"; ++ return "Get broker config by cluster or special broker."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java +index abe8fc622..1a8961e04 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java +@@ -38,7 +38,7 @@ public class GetBrokerEpochSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Fetch broker epoch entries"; ++ return "Fetch broker epoch entries."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java +index 7c54e650c..34b3ba7d3 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java +@@ -47,7 +47,7 @@ public class GetColdDataFlowCtrInfoSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "get cold data flow ctr info"; ++ return "Get cold data flow ctr info."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java +index b0477924f..f20407480 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java +@@ -36,7 +36,7 @@ public class RemoveColdDataFlowCtrGroupConfigSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "remove consumer from cold ctr config"; ++ return "Remove consumer from cold ctr config."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java +index b2ac48c84..90451b51f 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java +@@ -33,7 +33,7 @@ public class ResetMasterFlushOffsetSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Reset master flush offset in slave"; ++ return "Reset master flush offset in slave."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java +index 98abeb6ae..62816ef03 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java +@@ -37,7 +37,7 @@ public class UpdateBrokerConfigSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Update broker's config"; ++ return "Update broker's config."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java +index d06a24b57..8d1a00077 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java +@@ -39,7 +39,7 @@ public class UpdateColdDataFlowCtrGroupConfigSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "addOrUpdate cold data flow ctr group config"; ++ return "Add or update cold data flow ctr group config."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +index 7253970bd..d755e9e5d 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +@@ -48,7 +48,7 @@ public class CLusterSendMsgRTCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "List All clusters Message Send RT"; ++ return "List All clusters Message Send RT."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java +index a7a840a44..ede0fa5cf 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java +@@ -41,7 +41,7 @@ public class ClusterListSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "List cluster infos"; ++ return "List cluster infos."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java +index 630961e31..35f73d8a0 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java +@@ -39,7 +39,7 @@ public class ConsumerConnectionSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Query consumer's socket connection, client version and subscription"; ++ return "Query consumer's socket connection, client version and subscription."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java +index 2533982c8..bde674ab2 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java +@@ -36,7 +36,7 @@ public class ProducerConnectionSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Query producer's socket connection and client version"; ++ return "Query producer's socket connection and client version."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java +index 72b9c975e..d8f6f9aa9 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java +@@ -47,7 +47,7 @@ public class ConsumerStatusSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Query consumer's internal data structure"; ++ return "Query consumer's internal data structure."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java +index 6095e7668..4a8253a02 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java +@@ -43,7 +43,7 @@ public class GetConsumerConfigSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Get consumer config by subscription group name"; ++ return "Get consumer config by subscription group name."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java +index 2d08d0bd0..f5e140433 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java +@@ -34,7 +34,7 @@ public class StartMonitoringSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Start Monitoring"; ++ return "Start Monitoring."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java +index f87bafc93..b17da4de4 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java +@@ -41,7 +41,7 @@ public class UpdateSubGroupSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Update or create subscription group"; ++ return "Update or create subscription group."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java +index e9e5be4a5..007d42ae6 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java +@@ -33,7 +33,7 @@ public class AddBrokerSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Add a broker to specified container"; ++ return "Add a broker to specified container."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java +index 7c455f858..ab25d8ebe 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java +@@ -33,7 +33,7 @@ public class RemoveBrokerSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Remove a broker from specified container"; ++ return "Remove a broker from specified container."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java +index 856e4b426..24ed02566 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java +@@ -37,7 +37,7 @@ public class CleanControllerBrokerMetaSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Clean metadata of broker on controller"; ++ return "Clean metadata of broker on controller."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java +index 70bd7f8e9..966443127 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java +@@ -34,7 +34,7 @@ public class GetControllerMetaDataSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Get controller cluster's metadata"; ++ return "Get controller cluster's metadata."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java +index 1affe81f9..a522a903d 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java +@@ -37,7 +37,7 @@ public class ReElectMasterSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Re-elect the specified broker as master"; ++ return "Re-elect the specified broker as master."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java +index b8191296d..03613b29c 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java +@@ -42,7 +42,7 @@ public class ExportConfigsCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Export configs"; ++ return "Export configs."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java +index 1f9cf7d96..748f7b16e 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java +@@ -46,7 +46,7 @@ public class ExportMetadataCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Export metadata"; ++ return "Export metadata."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java +index a793b4b84..5d8bb37ba 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java +@@ -56,7 +56,7 @@ public class ExportMetricsCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Export metrics"; ++ return "Export metrics."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java +index 44b3ec3e1..b6231e4f9 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java +@@ -40,7 +40,7 @@ public class GetSyncStateSetSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Fetch syncStateSet for target brokers"; ++ return "Fetch syncStateSet for target brokers."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java +index b1795e046..931658a08 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java +@@ -41,7 +41,7 @@ public class HAStatusSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Fetch ha runtime status data"; ++ return "Fetch ha runtime status data."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java +index 4c6d5ffb6..b15b59d50 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java +@@ -40,7 +40,7 @@ public class CheckMsgSendRTCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Check message send response time"; ++ return "Check message send response time."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java +index 8aed59ea4..02ff53269 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java +@@ -70,7 +70,7 @@ public class ConsumeMessageCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Consume message"; ++ return "Consume message."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java +index ae6d9bdcf..eee8f3d4b 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java +@@ -38,7 +38,7 @@ import java.nio.file.Paths; + public class DumpCompactionLogCommand implements SubCommand { + @Override + public String commandDesc() { +- return "parse compaction log to message"; ++ return "Parse compaction log to message."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java +index 654560167..0418e88a7 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java +@@ -108,7 +108,7 @@ public class PrintMessageByQueueCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Print Message Detail"; ++ return "Print Message Detail by queueId."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java +index d01c36d42..bb82f5079 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java +@@ -62,7 +62,7 @@ public class PrintMessageSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Print Message Detail"; ++ return "Print Message Detail."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java +index 2880477f1..b42612150 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java +@@ -186,7 +186,7 @@ public class QueryMsgByIdSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Query Message by Id"; ++ return "Query Message by Id."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java +index ba7b00c3b..64627fd19 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java +@@ -36,7 +36,7 @@ public class QueryMsgByKeySubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Query Message by Key"; ++ return "Query Message by Key."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java +index d27313af1..14d0625fd 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java +@@ -39,7 +39,7 @@ public class QueryMsgByOffsetSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Query Message by offset"; ++ return "Query Message by offset."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +index 1b28f8be1..b71cee901 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +@@ -141,7 +141,7 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Query Message by Unique key"; ++ return "Query Message by Unique key."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java +index 2b982efef..2c546ec56 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java +@@ -65,7 +65,7 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Query a message trace"; ++ return "Query a message trace."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java +index 836ee192b..970da6b16 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java +@@ -41,7 +41,7 @@ public class SendMessageCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Send a message"; ++ return "Send a message."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java +index 98542d065..0b0a075bd 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java +@@ -34,7 +34,7 @@ public class AddWritePermSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Add write perm of broker in all name server you defined in the -n param"; ++ return "Add write perm of broker in all name server you defined in the -n param."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java +index 213931ed8..637dd52c8 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java +@@ -34,7 +34,7 @@ public class WipeWritePermSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Wipe write perm of broker in all name server you defined in the -n param"; ++ return "Wipe write perm of broker in all name server you defined in the -n param."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java +index 139821f9c..b22491a59 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java +@@ -41,7 +41,7 @@ public class SkipAccumulationSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Skip all messages that are accumulated (not consumed) currently"; ++ return "Skip all messages that are accumulated (not consumed) currently."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java +index 1d49bbe11..96097a93e 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java +@@ -144,7 +144,7 @@ public class StatsAllSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Topic and Consumer tps stats"; ++ return "Topic and Consumer tps stats."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java +index 3fa42f297..6a9b81eb8 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java +@@ -41,7 +41,7 @@ public class AllocateMQSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Allocate MQ"; ++ return "Allocate MQ."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java +index 1dab693d9..098f34ff0 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java +@@ -34,7 +34,7 @@ public class TopicClusterSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Get cluster info for topic"; ++ return "Get cluster info for topic."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java +index 346bac704..d9a279f80 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java +@@ -45,7 +45,7 @@ public class TopicListSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Fetch all topic list from name server"; ++ return "Fetch all topic list from name server."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java +index f2dabec4e..70949d388 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java +@@ -42,7 +42,7 @@ public class TopicRouteSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Examine topic route info"; ++ return "Examine topic route info."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java +index fdb249fab..a1619eced 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java +@@ -40,7 +40,7 @@ public class TopicStatusSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Examine topic Status info"; ++ return "Examine topic Status info."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java +index bebc646b4..3040d04c2 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java +@@ -36,7 +36,7 @@ public class UpdateOrderConfCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Create or update or delete order conf"; ++ return "Create or update or delete order conf."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java +index 85a18c654..3daeee86c 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java +@@ -48,7 +48,7 @@ public class UpdateStaticTopicSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Update or create static topic, which has fixed number of queues"; ++ return "Update or create static topic, which has fixed number of queues."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java +index aaa881538..d27cd1861 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java +@@ -44,7 +44,7 @@ public class UpdateTopicPermSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Update topic perm"; ++ return "Update topic perm."; + } + + @Override +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java +index b68463396..298914175 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java +@@ -42,7 +42,7 @@ public class UpdateTopicSubCommand implements SubCommand { + + @Override + public String commandDesc() { +- return "Update or create topic"; ++ return "Update or create topic."; + } + + @Override +-- +2.32.0.windows.2 + + +From 744167bd01fab6821b4d5ae1794dc845153d5156 Mon Sep 17 00:00:00 2001 +From: Ziyi Tan +Date: Wed, 23 Aug 2023 08:32:17 +0800 +Subject: [PATCH 2/8] [ISSUE #7142] Add command `RocksDBConfigToJson` to + inspect rocksdb content (#7180) + +* feat: add command `RocksDBConfigToJson` to inspect rocksdb content + +Signed-off-by: Ziy1-Tan + +* refactor: fix style + +--------- + +Signed-off-by: Ziy1-Tan +Co-authored-by: Ziy1-Tan +--- + .../tools/command/MQAdminStartup.java | 2 + + .../metadata/RocksDBConfigToJsonCommand.java | 118 ++++++++++++++++++ + .../metadata/KvConfigToJsonCommandTest.java | 65 ++++++++++ + 3 files changed, 185 insertions(+) + create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java + create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java + +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +index 890125ca0..324aa1856 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +@@ -80,6 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; + import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; + import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; + import org.apache.rocketmq.tools.command.message.SendMessageCommand; ++import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; + import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; + import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; + import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; +@@ -211,6 +212,7 @@ public class MQAdminStartup { + + initCommand(new ClusterListSubCommand()); + initCommand(new TopicListSubCommand()); ++ initCommand(new RocksDBConfigToJsonCommand()); + + initCommand(new UpdateKvConfigCommand()); + initCommand(new DeleteKvConfigCommand()); +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +new file mode 100644 +index 000000000..3053f4684 +--- /dev/null ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +@@ -0,0 +1,118 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.tools.command.metadata; ++ ++import com.alibaba.fastjson.JSONObject; ++import org.apache.commons.cli.CommandLine; ++import org.apache.commons.cli.Option; ++import org.apache.commons.cli.Options; ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.config.RocksDBConfigManager; ++import org.apache.rocketmq.common.utils.DataConverter; ++import org.apache.rocketmq.remoting.RPCHook; ++import org.apache.rocketmq.tools.command.SubCommand; ++import org.apache.rocketmq.tools.command.SubCommandException; ++ ++import java.io.File; ++import java.util.HashMap; ++import java.util.Map; ++ ++public class RocksDBConfigToJsonCommand implements SubCommand { ++ private static final String TOPICS_JSON_CONFIG = "topics"; ++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; ++ ++ @Override ++ public String commandName() { ++ return "rocksDBConfigToJson"; ++ } ++ ++ @Override ++ public String commandDesc() { ++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; ++ } ++ ++ @Override ++ public Options buildCommandlineOptions(Options options) { ++ Option pathOption = new Option("p", "path", true, ++ "Absolute path to the metadata directory"); ++ pathOption.setRequired(true); ++ options.addOption(pathOption); ++ ++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + ++ "topics/subscriptionGroups"); ++ configTypeOption.setRequired(true); ++ options.addOption(configTypeOption); ++ ++ return options; ++ } ++ ++ @Override ++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { ++ String path = commandLine.getOptionValue("path").trim(); ++ if (StringUtils.isEmpty(path) || !new File(path).exists()) { ++ System.out.print("Rocksdb path is invalid.\n"); ++ return; ++ } ++ ++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); ++ ++ final long memTableFlushInterval = 60 * 60 * 1000L; ++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); ++ try { ++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for topics.json ++ final Map topicsJsonConfig = new HashMap<>(); ++ final Map topicConfigTable = new HashMap<>(); ++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++ final String topic = new String(key, DataConverter.charset); ++ final String topicConfig = new String(value, DataConverter.charset); ++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig); ++ topicConfigTable.put(topic, jsonObject); ++ }); ++ ++ if (isLoad) { ++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); ++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); ++ System.out.print(topicsJsonStr + "\n"); ++ return; ++ } ++ } ++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for subscriptionGroup.json ++ final Map subscriptionGroupJsonConfig = new HashMap<>(); ++ final Map subscriptionGroupTable = new HashMap<>(); ++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++ final String subscriptionGroup = new String(key, DataConverter.charset); ++ final String subscriptionGroupConfig = new String(value, DataConverter.charset); ++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); ++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); ++ }); ++ ++ if (isLoad) { ++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", ++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); ++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); ++ System.out.print(subscriptionGroupJsonStr + "\n"); ++ return; ++ } ++ } ++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); ++ } finally { ++ kvConfigManager.stop(); ++ } ++ } ++} +diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java +new file mode 100644 +index 000000000..b2f66c7b0 +--- /dev/null ++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java +@@ -0,0 +1,65 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.tools.command.metadata; ++ ++import org.apache.commons.cli.CommandLine; ++import org.apache.commons.cli.DefaultParser; ++import org.apache.commons.cli.Options; ++import org.apache.rocketmq.srvutil.ServerUtil; ++import org.apache.rocketmq.tools.command.SubCommandException; ++import org.junit.Test; ++ ++import java.io.File; ++ ++import static org.assertj.core.api.Assertions.assertThat; ++ ++public class KvConfigToJsonCommandTest { ++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; ++ ++ @Test ++ public void testExecute() throws SubCommandException { ++ { ++ String[] cases = new String[]{"topics", "subscriptionGroups"}; ++ for (String c : cases) { ++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); ++ Options options = ServerUtil.buildCommandlineOptions(new Options()); ++ String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c}; ++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, ++ cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.execute(commandLine, options, null); ++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c); ++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c); ++ } ++ } ++ // invalid cases ++ { ++ String[][] cases = new String[][]{ ++ {"-p " + BASE_PATH + "tmpPath", "-t topics"}, ++ {"-p ", "-t topics"}, ++ {"-p " + BASE_PATH + "topics", "-t invalid_type"} ++ }; ++ ++ for (String[] c : cases) { ++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); ++ Options options = ServerUtil.buildCommandlineOptions(new Options()); ++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, ++ cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.execute(commandLine, options, null); ++ } ++ } ++ } ++} +-- +2.32.0.windows.2 + + +From bdede35db365a49b211cdc249c68b0f60a3df46d Mon Sep 17 00:00:00 2001 +From: mxsm +Date: Wed, 23 Aug 2023 08:34:56 +0800 +Subject: [PATCH 3/8] [ISSUE #7124] Fix the typos in the code comments (#7125) + +--- + .../apache/rocketmq/broker/processor/ReplyMessageProcessor.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +index b2db356c8..d3bb048f7 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +@@ -234,7 +234,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor { + } else { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); +- //set to zore to avoid client decoding exception ++ //set to zero to avoid client decoding exception + responseHeader.setMsgId("0"); + responseHeader.setQueueId(queueIdInt); + responseHeader.setQueueOffset(0L); +-- +2.32.0.windows.2 + + +From 9bb73b9a38548b99ac5126c40380c3c2e7fc586e Mon Sep 17 00:00:00 2001 +From: lizhimins <707364882@qq.com> +Date: Wed, 23 Aug 2023 09:46:27 +0800 +Subject: [PATCH 4/8] [#ISSUE 7222] Bug fix and refactoring of the Indexfile in + tiered storage (#7224) + +--- + .../tieredstore/file/TieredIndexFile.java | 38 +++++++-- + .../tieredstore/file/TieredIndexFileTest.java | 84 +++++-------------- + 2 files changed, 52 insertions(+), 70 deletions(-) + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java +index 50beb01ae..eda5e0106 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java +@@ -16,6 +16,7 @@ + */ + package org.apache.rocketmq.tieredstore.file; + ++import com.google.common.annotations.VisibleForTesting; + import java.io.File; + import java.io.IOException; + import java.nio.ByteBuffer; +@@ -99,7 +100,7 @@ public class TieredIndexFile { + this::doScheduleTask, 10, 10, TimeUnit.SECONDS); + } + +- private void doScheduleTask() { ++ protected void doScheduleTask() { + try { + curFileLock.lock(); + try { +@@ -145,6 +146,11 @@ public class TieredIndexFile { + } + } + ++ @VisibleForTesting ++ public MappedFile getPreMappedFile() { ++ return preMappedFile; ++ } ++ + private void initFile() throws IOException { + curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); + initIndexFileHeader(curMappedFile); +@@ -156,19 +162,26 @@ public class TieredIndexFile { + + if (isFileSealed(curMappedFile)) { + if (preFileExists) { +- preFile.delete(); ++ if (preFile.delete()) { ++ logger.info("Pre IndexFile deleted success", preFilepath); ++ } else { ++ logger.error("Pre IndexFile deleted failed", preFilepath); ++ } + } + boolean rename = curMappedFile.renameTo(preFilepath); + if (rename) { + preMappedFile = curMappedFile; + curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); ++ initIndexFileHeader(curMappedFile); + preFileExists = true; + } + } ++ + if (preFileExists) { + synchronized (TieredIndexFile.class) { + if (inflightCompactFuture.isDone()) { +- inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(new CompactTask(storeConfig, preMappedFile, flatFile), null); ++ inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit( ++ new CompactTask(storeConfig, preMappedFile, flatFile), null); + } + } + } +@@ -261,7 +274,8 @@ public class TieredIndexFile { + } + } + +- public CompletableFuture>> queryAsync(String topic, String key, long beginTime, long endTime) { ++ public CompletableFuture>> queryAsync(String topic, String key, long beginTime, ++ long endTime) { + int hashCode = indexKeyHashMethod(buildKey(topic, key)); + int slotPosition = hashCode % maxHashSlotNum; + List fileSegmentList = flatFile.getFileListByTime(beginTime, endTime); +@@ -355,7 +369,7 @@ public class TieredIndexFile { + private final int fileMaxSize; + private MappedFile originFile; + private TieredFlatFile fileQueue; +- private final MappedFile compactFile; ++ private MappedFile compactFile; + + public CompactTask(TieredMessageStoreConfig storeConfig, MappedFile originFile, + TieredFlatFile fileQueue) throws IOException { +@@ -381,6 +395,17 @@ public class TieredIndexFile { + } catch (Throwable throwable) { + logger.error("TieredIndexFile#compactTask: compact index file failed:", throwable); + } ++ ++ try { ++ if (originFile != null) { ++ originFile.destroy(-1); ++ } ++ if (compactFile != null) { ++ compactFile.destroy(-1); ++ } ++ } catch (Throwable throwable) { ++ logger.error("TieredIndexFile#compactTask: destroy index file failed:", throwable); ++ } + } + + public void compact() { +@@ -396,6 +421,8 @@ public class TieredIndexFile { + fileQueue.commit(true); + compactFile.destroy(-1); + originFile.destroy(-1); ++ compactFile = null; ++ originFile = null; + } + + private void buildCompactFile() { +@@ -414,6 +441,7 @@ public class TieredIndexFile { + if (slotValue != -1) { + int indexTotalSize = 0; + int indexPosition = slotValue; ++ + while (indexPosition >= 0 && indexPosition < maxIndexNum) { + int indexOffset = INDEX_FILE_HEADER_SIZE + maxHashSlotNum * INDEX_FILE_HASH_SLOT_SIZE + + indexPosition * INDEX_FILE_HASH_ORIGIN_INDEX_SIZE; +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +index 7ef49578d..262d6645b 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +@@ -19,9 +19,8 @@ package org.apache.rocketmq.tieredstore.file; + import com.sun.jna.Platform; + import java.io.IOException; + import java.nio.ByteBuffer; ++import java.time.Duration; + import java.util.List; +-import java.util.concurrent.TimeUnit; +-import org.apache.commons.lang3.SystemUtils; + import org.apache.commons.lang3.tuple.Pair; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; +@@ -31,9 +30,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + import org.awaitility.Awaitility; + import org.junit.After; + import org.junit.Assert; +-import org.junit.Assume; + import org.junit.Before; +-import org.junit.Ignore; + import org.junit.Test; + + public class TieredIndexFileTest { +@@ -45,11 +42,12 @@ public class TieredIndexFileTest { + @Before + public void setUp() { + storeConfig = new TieredMessageStoreConfig(); ++ storeConfig.setBrokerName("IndexFileBroker"); + storeConfig.setStorePathRootDir(storePath); +- storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment"); +- storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2); +- storeConfig.setTieredStoreIndexFileMaxIndexNum(3); +- mq = new MessageQueue("TieredIndexFileTest", storeConfig.getBrokerName(), 1); ++ storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment"); ++ storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5); ++ storeConfig.setTieredStoreIndexFileMaxIndexNum(20); ++ mq = new MessageQueue("IndexFileTest", storeConfig.getBrokerName(), 1); + TieredStoreUtil.getMetadataStore(storeConfig); + TieredStoreExecutor.init(); + } +@@ -61,77 +59,33 @@ public class TieredIndexFileTest { + TieredStoreExecutor.shutdown(); + } + +- @Ignore + @Test + public void testAppendAndQuery() throws IOException, ClassNotFoundException, NoSuchMethodException { + if (Platform.isWindows()) { + return; + } + +- // skip this test on windows +- Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); +- + TieredFileAllocator fileQueueFactory = new TieredFileAllocator(storeConfig); + TieredIndexFile indexFile = new TieredIndexFile(fileQueueFactory, storePath); ++ + indexFile.append(mq, 0, "key3", 3, 300, 1000); + indexFile.append(mq, 0, "key2", 2, 200, 1100); + indexFile.append(mq, 0, "key1", 1, 100, 1200); + +- Awaitility.waitAtMost(5, TimeUnit.SECONDS) +- .until(() -> { +- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); +- if (indexList.size() != 1) { +- return false; +- } +- +- ByteBuffer indexBuffer = indexList.get(0).getValue(); +- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 2, indexBuffer.remaining()); +- +- Assert.assertEquals(1, indexBuffer.getLong(4 + 4 + 4)); +- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8)); +- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +- +- Assert.assertEquals(3, indexBuffer.getLong(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4)); +- Assert.assertEquals(300, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8)); +- Assert.assertEquals(0, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8 + 4)); +- return true; +- }); +- +- indexFile.append(mq, 0, "key4", 4, 400, 1300); +- indexFile.append(mq, 0, "key4", 4, 400, 1300); +- indexFile.append(mq, 0, "key4", 4, 400, 1300); +- +- Awaitility.waitAtMost(5, TimeUnit.SECONDS) +- .until(() -> { +- List> indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1300, 1300).join(); +- if (indexList.size() != 1) { +- return false; +- } +- +- ByteBuffer indexBuffer = indexList.get(0).getValue(); +- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); +- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); +- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); +- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +- return true; +- }); +- +- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1300, 1300).join(); ++ // do not do schedule task here ++ TieredStoreExecutor.shutdown(); ++ List> indexList = ++ indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); + Assert.assertEquals(0, indexList.size()); + +- indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1200, 1300).join(); +- Assert.assertEquals(2, indexList.size()); +- +- ByteBuffer indexBuffer = indexList.get(0).getValue(); +- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); +- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); +- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); +- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++ // do compaction once ++ TieredStoreExecutor.init(); ++ storeConfig.setTieredStoreIndexFileRollingIdleInterval(0); ++ indexFile.doScheduleTask(); ++ Awaitility.await().atMost(Duration.ofSeconds(10)) ++ .until(() -> !indexFile.getPreMappedFile().getFile().exists()); + +- indexBuffer = indexList.get(1).getValue(); +- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE, indexBuffer.remaining()); +- Assert.assertEquals(2, indexBuffer.getLong(4 + 4 + 4)); +- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8)); +- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++ Assert.assertEquals(1, indexList.size()); + } + } +-- +2.32.0.windows.2 + + +From 69c26d3d29cde7b4484ecd112ab9224f9f42bf45 Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Wed, 23 Aug 2023 10:27:52 +0800 +Subject: [PATCH 5/8] [ISSUE #7228] Converge the use of some important + variables for some class + +--- + .../apache/rocketmq/store/ConsumeQueue.java | 16 ++++++------ + .../rocketmq/store/MappedFileQueue.java | 26 +++++++++++-------- + .../store/MultiPathMappedFileQueue.java | 4 +-- + 3 files changed, 24 insertions(+), 22 deletions(-) + +diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +index a0b886eb0..56bee2af3 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +@@ -145,7 +145,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + if (offset >= 0 && size > 0) { + mappedFileOffset = i + CQ_STORE_UNIT_SIZE; +- this.maxPhysicOffset = offset + size; ++ this.setMaxPhysicOffset(offset + size); + if (isExtAddr(tagsCode)) { + maxExtAddr = tagsCode; + } +@@ -409,7 +409,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + int logicFileSize = this.mappedFileSize; + +- this.maxPhysicOffset = phyOffset; ++ this.setMaxPhysicOffset(phyOffset); + long maxExtAddr = 1; + boolean shouldDeleteFile = false; + while (true) { +@@ -435,7 +435,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + mappedFile.setWrotePosition(pos); + mappedFile.setCommittedPosition(pos); + mappedFile.setFlushedPosition(pos); +- this.maxPhysicOffset = offset + size; ++ this.setMaxPhysicOffset(offset + size); + // This maybe not take effect, when not every consume queue has extend file. + if (isExtAddr(tagsCode)) { + maxExtAddr = tagsCode; +@@ -453,7 +453,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + mappedFile.setWrotePosition(pos); + mappedFile.setCommittedPosition(pos); + mappedFile.setFlushedPosition(pos); +- this.maxPhysicOffset = offset + size; ++ this.setMaxPhysicOffset(offset + size); + if (isExtAddr(tagsCode)) { + maxExtAddr = tagsCode; + } +@@ -881,8 +881,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, + final long cqOffset) { + +- if (offset + size <= this.maxPhysicOffset) { +- log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset); ++ if (offset + size <= this.getMaxPhysicOffset()) { ++ log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", this.getMaxPhysicOffset(), offset); + return true; + } + +@@ -926,7 +926,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + ); + } + } +- this.maxPhysicOffset = offset + size; ++ this.setMaxPhysicOffset(offset + size); + return mappedFile.appendMessage(this.byteBufferIndex.array()); + } + return false; +@@ -1130,7 +1130,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + @Override + public void destroy() { +- this.maxPhysicOffset = -1; ++ this.setMaxPhysicOffset(-1); + this.minLogicOffset = 0; + this.mappedFileQueue.destroy(); + if (isExtReadEnable()) { +diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +index 0bc70642f..32b90d14f 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +@@ -285,7 +285,7 @@ public class MappedFileQueue implements Swappable { + if (this.mappedFiles.isEmpty()) + return 0; + +- long committed = this.flushedWhere; ++ long committed = this.getFlushedWhere(); + if (committed != 0) { + MappedFile mappedFile = this.getLastMappedFile(0, false); + if (mappedFile != null) { +@@ -442,11 +442,11 @@ public class MappedFileQueue implements Swappable { + } + + public long remainHowManyDataToCommit() { +- return getMaxWrotePosition() - committedWhere; ++ return getMaxWrotePosition() - getCommittedWhere(); + } + + public long remainHowManyDataToFlush() { +- return getMaxOffset() - flushedWhere; ++ return getMaxOffset() - this.getFlushedWhere(); + } + + public void deleteLastMappedFile() { +@@ -616,15 +616,15 @@ public class MappedFileQueue implements Swappable { + + public boolean flush(final int flushLeastPages) { + boolean result = true; +- MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0); ++ MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0); + if (mappedFile != null) { + long tmpTimeStamp = mappedFile.getStoreTimestamp(); + int offset = mappedFile.flush(flushLeastPages); + long where = mappedFile.getFileFromOffset() + offset; +- result = where == this.flushedWhere; +- this.flushedWhere = where; ++ result = where == this.getFlushedWhere(); ++ this.setFlushedWhere(where); + if (0 == flushLeastPages) { +- this.storeTimestamp = tmpTimeStamp; ++ this.setStoreTimestamp(tmpTimeStamp); + } + } + +@@ -633,12 +633,12 @@ public class MappedFileQueue implements Swappable { + + public synchronized boolean commit(final int commitLeastPages) { + boolean result = true; +- MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0); ++ MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0); + if (mappedFile != null) { + int offset = mappedFile.commit(commitLeastPages); + long where = mappedFile.getFileFromOffset() + offset; +- result = where == this.committedWhere; +- this.committedWhere = where; ++ result = where == this.getCommittedWhere(); ++ this.setCommittedWhere(where); + } + + return result; +@@ -763,7 +763,7 @@ public class MappedFileQueue implements Swappable { + mf.destroy(1000 * 3); + } + this.mappedFiles.clear(); +- this.flushedWhere = 0; ++ this.setFlushedWhere(0); + + // delete parent directory + File file = new File(storePath); +@@ -848,6 +848,10 @@ public class MappedFileQueue implements Swappable { + return storeTimestamp; + } + ++ public void setStoreTimestamp(long storeTimestamp) { ++ this.storeTimestamp = storeTimestamp; ++ } ++ + public List getMappedFiles() { + return mappedFiles; + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java +index 8f5af9438..8ff050dfe 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java +@@ -16,7 +16,6 @@ + */ + package org.apache.rocketmq.store; + +- + import java.util.Arrays; + import java.util.HashSet; + import java.util.Set; +@@ -113,8 +112,7 @@ public class MultiPathMappedFileQueue extends MappedFileQueue { + mf.destroy(1000 * 3); + } + this.mappedFiles.clear(); +- this.flushedWhere = 0; +- ++ this.setFlushedWhere(0); + + Set storePathSet = getPaths(); + storePathSet.addAll(getReadonlyPaths()); +-- +2.32.0.windows.2 + + +From 3884f595949462044c5cb3c236199bc1d7ad2341 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= + +Date: Wed, 23 Aug 2023 11:10:30 +0800 +Subject: [PATCH 6/8] [ISSUE #7149] When creating and updating Topic, there + will be problems with permission settings (#7151) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings + +* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings + +* [issue#7249] + +--------- + +Co-authored-by: 十真 +--- + .../main/java/org/apache/rocketmq/broker/BrokerController.java | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index 13f9d002b..e8f943702 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -1733,7 +1733,8 @@ public class BrokerController { + new TopicConfig(topicConfig.getTopicName(), + topicConfig.getReadQueueNums(), + topicConfig.getWriteQueueNums(), +- this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); ++ topicConfig.getPerm() ++ & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); + } else { + registerTopicConfig = new TopicConfig(topicConfig); + } +-- +2.32.0.windows.2 + + +From 017ad110475e8024585327b44f47e5e97aabc63b Mon Sep 17 00:00:00 2001 +From: echooymxq +Date: Wed, 23 Aug 2023 11:11:42 +0800 +Subject: [PATCH 7/8] [ISSUE #7219] Fix Concurrent modify syncStateSet and Mark + synchronizing frequently when shrink. (#7220) + +--- + .../broker/controller/ReplicasManager.java | 29 ++++++++++--------- + .../ha/autoswitch/AutoSwitchHAService.java | 21 ++++++++------ + 2 files changed, 28 insertions(+), 22 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +index abae7cdb0..37c82e434 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +@@ -542,7 +542,7 @@ public class ReplicasManager { + this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId()); + this.tempBrokerMetadata.clear(); + this.brokerControllerId = this.brokerMetadata.getBrokerId(); +- this.haService.setBrokerControllerId(this.brokerControllerId); ++ this.haService.setLocalBrokerId(this.brokerControllerId); + return true; + } catch (Exception e) { + LOGGER.error("fail to create metadata file", e); +@@ -594,7 +594,7 @@ public class ReplicasManager { + if (this.brokerMetadata.isLoaded()) { + this.registerState = RegisterState.CREATE_METADATA_FILE_DONE; + this.brokerControllerId = brokerMetadata.getBrokerId(); +- this.haService.setBrokerControllerId(this.brokerControllerId); ++ this.haService.setLocalBrokerId(this.brokerControllerId); + return; + } + // 2. check if temp metadata exist +@@ -735,23 +735,26 @@ public class ReplicasManager { + if (this.checkSyncStateSetTaskFuture != null) { + this.checkSyncStateSetTaskFuture.cancel(false); + } +- this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(() -> { +- checkSyncStateSetAndDoReport(); +- }, 3 * 1000, this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); ++ this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000, ++ this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); + } + + private void checkSyncStateSetAndDoReport() { +- final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); +- newSyncStateSet.add(this.brokerControllerId); +- synchronized (this) { +- if (this.syncStateSet != null) { +- // Check if syncStateSet changed +- if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { +- return; ++ try { ++ final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); ++ newSyncStateSet.add(this.brokerControllerId); ++ synchronized (this) { ++ if (this.syncStateSet != null) { ++ // Check if syncStateSet changed ++ if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { ++ return; ++ } + } + } ++ doReportSyncStateSetChanged(newSyncStateSet); ++ } catch (Exception e) { ++ LOGGER.error("Check syncStateSet error", e); + } +- doReportSyncStateSetChanged(newSyncStateSet); + } + + private void doReportSyncStateSetChanged(Set newSyncStateSet) { +diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +index 6dc734e0c..d5393fdca 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +@@ -41,6 +41,7 @@ import java.nio.channels.SocketChannel; + import java.util.ArrayList; + import java.util.HashSet; + import java.util.List; ++import java.util.Iterator; + import java.util.Map; + import java.util.Objects; + import java.util.Set; +@@ -73,7 +74,7 @@ public class AutoSwitchHAService extends DefaultHAService { + private EpochFileCache epochCache; + private AutoSwitchHAClient haClient; + +- private Long brokerControllerId = null; ++ private Long localBrokerId = null; + + public AutoSwitchHAService() { + } +@@ -287,9 +288,11 @@ public class AutoSwitchHAService extends DefaultHAService { + + // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable, + // it means that the broker has not connected. +- for (Long slaveBrokerId : newSyncStateSet) { +- if (!this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { +- newSyncStateSet.remove(slaveBrokerId); ++ Iterator iterator = newSyncStateSet.iterator(); ++ while (iterator.hasNext()) { ++ Long slaveBrokerId = iterator.next(); ++ if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { ++ iterator.remove(); + isSyncStateSetChanged = true; + } + } +@@ -419,7 +422,7 @@ public class AutoSwitchHAService extends DefaultHAService { + // To avoid the syncStateSet is not consistent with connectionList. + // Fix issue: https://github.com/apache/rocketmq/issues/6662 + for (Long syncId : currentSyncStateSet) { +- if (!idList.contains(syncId) && this.brokerControllerId != null && !Objects.equals(syncId, this.brokerControllerId)) { ++ if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) { + LOGGER.warn("Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.", syncId); + // Without check and re-compute, return the confirmOffset's value directly. + return this.defaultMessageStore.getConfirmOffsetDirectly(); +@@ -545,12 +548,12 @@ public class AutoSwitchHAService extends DefaultHAService { + return this.epochCache.getAllEntries(); + } + +- public Long getBrokerControllerId() { +- return brokerControllerId; ++ public Long getLocalBrokerId() { ++ return localBrokerId; + } + +- public void setBrokerControllerId(Long brokerControllerId) { +- this.brokerControllerId = brokerControllerId; ++ public void setLocalBrokerId(Long localBrokerId) { ++ this.localBrokerId = localBrokerId; + } + + class AutoSwitchAcceptSocketService extends AcceptSocketService { +-- +2.32.0.windows.2 + + +From 77e8e54b37c3fc3ea0beffc1ace6f5bf20af10d9 Mon Sep 17 00:00:00 2001 +From: lk +Date: Wed, 23 Aug 2023 15:56:39 +0800 +Subject: [PATCH 8/8] [ISSUE #7223] Support batch ack for grpc client in proxy + (#7225) + +--- + .../client/impl/mqclient/MQClientAPIExt.java | 26 +++ + .../rocketmq/proxy/config/ProxyConfig.java | 10 + + .../grpc/v2/consumer/AckMessageActivity.java | 136 ++++++++--- + .../proxy/processor/AbstractProcessor.java | 4 +- + .../proxy/processor/BatchAckResult.java | 53 +++++ + .../proxy/processor/ConsumerProcessor.java | 64 +++++ + .../processor/DefaultMessagingProcessor.java | 7 + + .../proxy/processor/MessagingProcessor.java | 18 ++ + .../message/ClusterMessageService.java | 16 +- + .../service/message/LocalMessageService.java | 58 +++++ + .../proxy/service/message/MessageService.java | 8 + + .../service/message/ReceiptHandleMessage.java | 39 ++++ + .../v2/consumer/AckMessageActivityTest.java | 221 +++++++++++++++--- + .../proxy/processor/BaseProcessorTest.java | 18 +- + .../processor/ConsumerProcessorTest.java | 115 +++++++++ + .../service/mqclient/MQClientAPIExtTest.java | 12 + + 16 files changed, 728 insertions(+), 77 deletions(-) + create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java + create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java + +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +index fb8f8d11f..d7c8ef8d9 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +@@ -306,6 +306,32 @@ public class MQClientAPIExt extends MQClientAPIImpl { + return future; + } + ++ public CompletableFuture batchAckMessageAsync( ++ String brokerAddr, ++ String topic, ++ String consumerGroup, ++ List extraInfoList, ++ long timeoutMillis ++ ) { ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++ this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() { ++ @Override ++ public void onSuccess(AckResult ackResult) { ++ future.complete(ackResult); ++ } ++ ++ @Override ++ public void onException(Throwable t) { ++ future.completeExceptionally(t); ++ } ++ }, topic, consumerGroup, extraInfoList); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++ return future; ++ } ++ + public CompletableFuture changeInvisibleTimeAsync( + String brokerAddr, + String brokerName, +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +index 39caaa0d9..76a243919 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +@@ -250,6 +250,8 @@ public class ProxyConfig implements ConfigFile { + private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000; + private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000; + ++ private boolean enableBatchAck = false; ++ + @Override + public void initData() { + parseDelayLevel(); +@@ -1379,4 +1381,12 @@ public class ProxyConfig implements ConfigFile { + public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) { + this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; + } ++ ++ public boolean isEnableBatchAck() { ++ return enableBatchAck; ++ } ++ ++ public void setEnableBatchAck(boolean enableBatchAck) { ++ this.enableBatchAck = enableBatchAck; ++ } + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java +index 9a3a77201..97c716c8f 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java +@@ -31,12 +31,15 @@ import org.apache.rocketmq.client.consumer.AckStatus; + import org.apache.rocketmq.common.consumer.ReceiptHandle; + import org.apache.rocketmq.proxy.common.MessageReceiptHandle; + import org.apache.rocketmq.proxy.common.ProxyContext; ++import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; + import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; + import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; + import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; + import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; ++import org.apache.rocketmq.proxy.processor.BatchAckResult; + import org.apache.rocketmq.proxy.processor.MessagingProcessor; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + + public class AckMessageActivity extends AbstractMessingActivity { + +@@ -50,60 +53,98 @@ public class AckMessageActivity extends AbstractMessingActivity { + + try { + validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); +- +- CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; +- for (int i = 0; i < request.getEntriesCount(); i++) { +- futures[i] = processAckMessage(ctx, request, request.getEntries(i)); ++ String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); ++ String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()); ++ if (ConfigurationManager.getProxyConfig().isEnableBatchAck()) { ++ future = ackMessageInBatch(ctx, group, topic, request); ++ } else { ++ future = ackMessageOneByOne(ctx, group, topic, request); + } +- CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +- if (throwable != null) { +- future.completeExceptionally(throwable); +- return; +- } ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++ return future; ++ } ++ ++ protected CompletableFuture ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) { ++ List handleMessageList = new ArrayList<>(request.getEntriesCount()); + ++ for (AckMessageEntry ackMessageEntry : request.getEntriesList()) { ++ String handleString = getHandleString(ctx, group, request, ackMessageEntry); ++ handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId())); ++ } ++ return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic) ++ .thenApply(batchAckResultList -> { ++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder(); + Set responseCodes = new HashSet<>(); +- List entryList = new ArrayList<>(); +- for (CompletableFuture entryFuture : futures) { +- AckMessageResultEntry entryResult = entryFuture.join(); +- responseCodes.add(entryResult.getStatus().getCode()); +- entryList.add(entryResult); +- } +- AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() +- .addAllEntries(entryList); +- if (responseCodes.size() > 1) { +- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); +- } else if (responseCodes.size() == 1) { +- Code code = responseCodes.stream().findAny().get(); +- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); +- } else { +- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); ++ for (BatchAckResult batchAckResult : batchAckResultList) { ++ AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult); ++ responseBuilder.addEntries(entry); ++ responseCodes.add(entry.getStatus().getCode()); + } +- future.complete(responseBuilder.build()); ++ setAckResponseStatus(responseBuilder, responseCodes); ++ return responseBuilder.build(); + }); +- } catch (Throwable t) { +- future.completeExceptionally(t); ++ } ++ ++ protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) { ++ ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage(); ++ AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder() ++ .setMessageId(handleMessage.getMessageId()) ++ .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle()); ++ if (batchAckResult.getProxyException() != null) { ++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException())); ++ } else { ++ AckResult ackResult = batchAckResult.getAckResult(); ++ if (AckStatus.OK.equals(ackResult.getStatus())) { ++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); ++ } else { ++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")); ++ } + } +- return future; ++ return resultBuilder.build(); + } + +- protected CompletableFuture processAckMessage(ProxyContext ctx, AckMessageRequest request, ++ protected CompletableFuture ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) { ++ CompletableFuture resultFuture = new CompletableFuture<>(); ++ CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; ++ for (int i = 0; i < request.getEntriesCount(); i++) { ++ futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i)); ++ } ++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { ++ if (throwable != null) { ++ resultFuture.completeExceptionally(throwable); ++ return; ++ } ++ ++ Set responseCodes = new HashSet<>(); ++ List entryList = new ArrayList<>(); ++ for (CompletableFuture entryFuture : futures) { ++ AckMessageResultEntry entryResult = entryFuture.join(); ++ responseCodes.add(entryResult.getStatus().getCode()); ++ entryList.add(entryResult); ++ } ++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() ++ .addAllEntries(entryList); ++ setAckResponseStatus(responseBuilder, responseCodes); ++ resultFuture.complete(responseBuilder.build()); ++ }); ++ return resultFuture; ++ } ++ ++ protected CompletableFuture processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request, + AckMessageEntry ackMessageEntry) { + CompletableFuture future = new CompletableFuture<>(); + + try { +- String handleString = ackMessageEntry.getReceiptHandle(); +- +- String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); +- MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); +- if (messageReceiptHandle != null) { +- handleString = messageReceiptHandle.getReceiptHandleStr(); +- } ++ String handleString = this.getHandleString(ctx, group, request, ackMessageEntry); + CompletableFuture ackResultFuture = this.messagingProcessor.ackMessage( + ctx, + ReceiptHandle.decode(handleString), + ackMessageEntry.getMessageId(), + group, +- GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); ++ topic ++ ); + ackResultFuture.thenAccept(result -> { + future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result)); + }).exceptionally(t -> { +@@ -139,4 +180,25 @@ public class AckMessageActivity extends AbstractMessingActivity { + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")) + .build(); + } ++ ++ protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set responseCodes) { ++ if (responseCodes.size() > 1) { ++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); ++ } else if (responseCodes.size() == 1) { ++ Code code = responseCodes.stream().findAny().get(); ++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); ++ } else { ++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); ++ } ++ } ++ ++ protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) { ++ String handleString = ackMessageEntry.getReceiptHandle(); ++ ++ MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); ++ if (messageReceiptHandle != null) { ++ handleString = messageReceiptHandle.getReceiptHandleStr(); ++ } ++ return handleString; ++ } + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java +index b61c3df9e..c63212c23 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java +@@ -27,6 +27,8 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { + protected MessagingProcessor messagingProcessor; + protected ServiceManager serviceManager; + ++ protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); ++ + public AbstractProcessor(MessagingProcessor messagingProcessor, + ServiceManager serviceManager) { + this.messagingProcessor = messagingProcessor; +@@ -35,7 +37,7 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { + + protected void validateReceiptHandle(ReceiptHandle handle) { + if (handle.isExpired()) { +- throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); ++ throw EXPIRED_HANDLE_PROXY_EXCEPTION; + } + } + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java +new file mode 100644 +index 000000000..dfb9c9b9e +--- /dev/null ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java +@@ -0,0 +1,53 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.proxy.processor; ++ ++import org.apache.rocketmq.client.consumer.AckResult; ++import org.apache.rocketmq.proxy.common.ProxyException; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ ++public class BatchAckResult { ++ ++ private final ReceiptHandleMessage receiptHandleMessage; ++ private AckResult ackResult; ++ private ProxyException proxyException; ++ ++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, ++ AckResult ackResult) { ++ this.receiptHandleMessage = receiptHandleMessage; ++ this.ackResult = ackResult; ++ } ++ ++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, ++ ProxyException proxyException) { ++ this.receiptHandleMessage = receiptHandleMessage; ++ this.proxyException = proxyException; ++ } ++ ++ public ReceiptHandleMessage getReceiptHandleMessage() { ++ return receiptHandleMessage; ++ } ++ ++ public AckResult getAckResult() { ++ return ackResult; ++ } ++ ++ public ProxyException getProxyException() { ++ return proxyException; ++ } ++} +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +index 656a6339d..f3522b374 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +@@ -48,6 +48,7 @@ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; + import org.apache.rocketmq.proxy.common.utils.FutureUtils; + import org.apache.rocketmq.proxy.common.utils.ProxyUtils; + import org.apache.rocketmq.proxy.service.ServiceManager; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; + import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; + import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; +@@ -241,6 +242,69 @@ public class ConsumerProcessor extends AbstractProcessor { + return FutureUtils.addExecutor(future, this.executor); + } + ++ public CompletableFuture> batchAckMessage( ++ ProxyContext ctx, ++ List handleMessageList, ++ String consumerGroup, ++ String topic, ++ long timeoutMillis ++ ) { ++ CompletableFuture> future = new CompletableFuture<>(); ++ try { ++ List batchAckResultList = new ArrayList<>(handleMessageList.size()); ++ Map> brokerHandleListMap = new HashMap<>(); ++ ++ for (ReceiptHandleMessage handleMessage : handleMessageList) { ++ if (handleMessage.getReceiptHandle().isExpired()) { ++ batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION)); ++ continue; ++ } ++ List brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>()); ++ brokerHandleList.add(handleMessage); ++ } ++ ++ if (brokerHandleListMap.isEmpty()) { ++ return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor); ++ } ++ Set>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet(); ++ CompletableFuture>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()]; ++ int futureIndex = 0; ++ for (Map.Entry> entry : brokerHandleListMapEntrySet) { ++ futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis); ++ } ++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { ++ if (throwable != null) { ++ future.completeExceptionally(throwable); ++ } ++ for (CompletableFuture> resultFuture : futures) { ++ batchAckResultList.addAll(resultFuture.join()); ++ } ++ future.complete(batchAckResultList); ++ }); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++ return FutureUtils.addExecutor(future, this.executor); ++ } ++ ++ protected CompletableFuture> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List handleMessageList, long timeoutMillis) { ++ return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis) ++ .thenApply(result -> { ++ List results = new ArrayList<>(); ++ for (ReceiptHandleMessage handleMessage : handleMessageList) { ++ results.add(new BatchAckResult(handleMessage, result)); ++ } ++ return results; ++ }) ++ .exceptionally(throwable -> { ++ List results = new ArrayList<>(); ++ for (ReceiptHandleMessage handleMessage : handleMessageList) { ++ results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable))); ++ } ++ return results; ++ }); ++ } ++ + public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, + String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +index 188cb7b9b..ba150051b 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +@@ -46,6 +46,7 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.config.ProxyConfig; + import org.apache.rocketmq.proxy.service.ServiceManager; + import org.apache.rocketmq.proxy.service.ServiceManagerFactory; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.apache.rocketmq.proxy.service.metadata.MetadataService; + import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; + import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; +@@ -183,6 +184,12 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen + return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis); + } + ++ @Override ++ public CompletableFuture> batchAckMessage(ProxyContext ctx, ++ List handleMessageList, String consumerGroup, String topic, long timeoutMillis) { ++ return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis); ++ } ++ + @Override + public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, + String groupName, String topicName, long invisibleTime, long timeoutMillis) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +index d86be0bd8..2ae7418ba 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +@@ -37,6 +37,7 @@ import org.apache.rocketmq.proxy.common.Address; + import org.apache.rocketmq.proxy.common.MessageReceiptHandle; + import org.apache.rocketmq.proxy.common.ProxyContext; + import org.apache.rocketmq.common.utils.StartAndShutdown; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.apache.rocketmq.proxy.service.metadata.MetadataService; + import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; + import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; +@@ -155,6 +156,23 @@ public interface MessagingProcessor extends StartAndShutdown { + long timeoutMillis + ); + ++ default CompletableFuture> batchAckMessage( ++ ProxyContext ctx, ++ List handleMessageList, ++ String consumerGroup, ++ String topic ++ ) { ++ return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS); ++ } ++ ++ CompletableFuture> batchAckMessage( ++ ProxyContext ctx, ++ List handleMessageList, ++ String consumerGroup, ++ String topic, ++ long timeoutMillis ++ ); ++ + default CompletableFuture changeInvisibleTime( + ProxyContext ctx, + ReceiptHandle handle, +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +index 9f163f1b9..70b72deae 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +@@ -20,9 +20,11 @@ import com.google.common.collect.Lists; + import java.util.List; + import java.util.Set; + import java.util.concurrent.CompletableFuture; ++import java.util.stream.Collectors; + import org.apache.rocketmq.client.consumer.AckResult; + import org.apache.rocketmq.client.consumer.PopResult; + import org.apache.rocketmq.client.consumer.PullResult; ++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; + import org.apache.rocketmq.client.producer.SendResult; + import org.apache.rocketmq.common.consumer.ReceiptHandle; + import org.apache.rocketmq.common.message.Message; +@@ -31,7 +33,6 @@ import org.apache.rocketmq.proxy.common.ProxyContext; + import org.apache.rocketmq.proxy.common.ProxyException; + import org.apache.rocketmq.proxy.common.ProxyExceptionCode; + import org.apache.rocketmq.proxy.common.utils.FutureUtils; +-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; + import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; + import org.apache.rocketmq.proxy.service.route.TopicRouteService; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; +@@ -137,6 +138,19 @@ public class ClusterMessageService implements MessageService { + ); + } + ++ @Override ++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, String consumerGroup, ++ String topic, long timeoutMillis) { ++ List extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList()); ++ return this.mqClientAPIFactory.getClient().batchAckMessageAsync( ++ this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()), ++ topic, ++ consumerGroup, ++ extraInfoList, ++ timeoutMillis ++ ); ++ } ++ + @Override + public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + PullMessageRequestHeader requestHeader, long timeoutMillis) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +index eb2c4d9ee..ca7dcc9eb 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.service.message; + import io.netty.channel.ChannelHandlerContext; + import java.nio.ByteBuffer; + import java.util.ArrayList; ++import java.util.BitSet; + import java.util.Collections; + import java.util.HashMap; + import java.util.List; +@@ -54,6 +55,8 @@ import org.apache.rocketmq.remoting.RPCHook; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; + import org.apache.rocketmq.remoting.protocol.RequestCode; + import org.apache.rocketmq.remoting.protocol.ResponseCode; ++import org.apache.rocketmq.remoting.protocol.body.BatchAck; ++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; + import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; + import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; + import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; +@@ -364,6 +367,61 @@ public class LocalMessageService implements MessageService { + }); + } + ++ @Override ++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, ++ String consumerGroup, String topic, long timeoutMillis) { ++ SimpleChannel channel = channelManager.createChannel(ctx); ++ ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); ++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); ++ ++ Map batchAckMap = new HashMap<>(); ++ for (ReceiptHandleMessage receiptHandleMessage : handleList) { ++ String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle(); ++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo); ++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + ++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" + ++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + ++ ExtraInfoUtil.getPopTime(extraInfoData); ++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { ++ BatchAck newBatchAck = new BatchAck(); ++ newBatchAck.setConsumerGroup(consumerGroup); ++ newBatchAck.setTopic(topic); ++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); ++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); ++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); ++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); ++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); ++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); ++ newBatchAck.setBitSet(new BitSet()); ++ return newBatchAck; ++ }); ++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); ++ } ++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); ++ requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName()); ++ requestBody.setAcks(new ArrayList<>(batchAckMap.values())); ++ ++ command.setBody(requestBody.encode()); ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++ RemotingCommand response = brokerController.getAckMessageProcessor() ++ .processRequest(channelHandlerContext, command); ++ future.complete(response); ++ } catch (Exception e) { ++ log.error("Fail to process batchAckMessage command", e); ++ future.completeExceptionally(e); ++ } ++ return future.thenApply(r -> { ++ AckResult ackResult = new AckResult(); ++ if (ResponseCode.SUCCESS == r.getCode()) { ++ ackResult.setStatus(AckStatus.OK); ++ } else { ++ ackResult.setStatus(AckStatus.NO_EXIST); ++ } ++ return ackResult; ++ }); ++ } ++ + @Override + public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + PullMessageRequestHeader requestHeader, long timeoutMillis) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +index 15da17154..58a835adb 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +@@ -91,6 +91,14 @@ public interface MessageService { + long timeoutMillis + ); + ++ CompletableFuture batchAckMessage( ++ ProxyContext ctx, ++ List handleList, ++ String consumerGroup, ++ String topic, ++ long timeoutMillis ++ ); ++ + CompletableFuture pullMessage( + ProxyContext ctx, + AddressableMessageQueue messageQueue, +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java +new file mode 100644 +index 000000000..ae63fed49 +--- /dev/null ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java +@@ -0,0 +1,39 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.proxy.service.message; ++ ++import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ ++public class ReceiptHandleMessage { ++ ++ private final ReceiptHandle receiptHandle; ++ private final String messageId; ++ ++ public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) { ++ this.receiptHandle = receiptHandle; ++ this.messageId = messageId; ++ } ++ ++ public ReceiptHandle getReceiptHandle() { ++ return receiptHandle; ++ } ++ ++ public String getMessageId() { ++ return messageId; ++ } ++} +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java +index 49fdfc6a8..3c4746105 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java +@@ -20,21 +20,32 @@ package org.apache.rocketmq.proxy.grpc.v2.consumer; + import apache.rocketmq.v2.AckMessageEntry; + import apache.rocketmq.v2.AckMessageRequest; + import apache.rocketmq.v2.AckMessageResponse; ++import apache.rocketmq.v2.AckMessageResultEntry; + import apache.rocketmq.v2.Code; + import apache.rocketmq.v2.Resource; ++import java.util.ArrayList; ++import java.util.HashMap; ++import java.util.List; ++import java.util.Map; + import java.util.concurrent.CompletableFuture; + import org.apache.rocketmq.client.consumer.AckResult; + import org.apache.rocketmq.client.consumer.AckStatus; + import org.apache.rocketmq.proxy.common.ProxyException; + import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; ++import org.apache.rocketmq.proxy.processor.BatchAckResult; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.junit.Before; + import org.junit.Test; ++import org.mockito.stubbing.Answer; + + import static org.junit.Assert.assertEquals; + import static org.mockito.ArgumentMatchers.any; ++import static org.mockito.ArgumentMatchers.anyList; + import static org.mockito.ArgumentMatchers.anyString; + import static org.mockito.ArgumentMatchers.eq; ++import static org.mockito.Mockito.doAnswer; + import static org.mockito.Mockito.when; + + public class AckMessageActivityTest extends BaseActivityTest { +@@ -52,43 +63,197 @@ public class AckMessageActivityTest extends BaseActivityTest { + + @Test + public void testAckMessage() throws Throwable { +- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString())) ++ ConfigurationManager.getProxyConfig().setEnableBatchAck(false); ++ ++ String msg1 = "msg1"; ++ String msg2 = "msg2"; ++ String msg3 = "msg3"; ++ ++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString())) + .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired")); + + AckResult msg2AckResult = new AckResult(); + msg2AckResult.setStatus(AckStatus.OK); +- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString())) ++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString())) + .thenReturn(CompletableFuture.completedFuture(msg2AckResult)); + + AckResult msg3AckResult = new AckResult(); + msg3AckResult.setStatus(AckStatus.NO_EXIST); +- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString())) ++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString())) + .thenReturn(CompletableFuture.completedFuture(msg3AckResult)); + +- AckMessageResponse response = this.ackMessageActivity.ackMessage( +- createContext(), +- AckMessageRequest.newBuilder() +- .setTopic(Resource.newBuilder().setName(TOPIC).build()) +- .setGroup(Resource.newBuilder().setName(GROUP).build()) +- .addEntries(AckMessageEntry.newBuilder() +- .setMessageId("msg1") +- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +- .build()) +- .addEntries(AckMessageEntry.newBuilder() +- .setMessageId("msg2") +- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +- .build()) +- .addEntries(AckMessageEntry.newBuilder() +- .setMessageId("msg3") +- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +- .build()) +- .build() +- ).get(); +- +- assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +- assertEquals(3, response.getEntriesCount()); +- assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); +- assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); +- assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg1) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg2) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.OK, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg3) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg1) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++ .build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg2) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(msg3) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ ++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); ++ assertEquals(3, response.getEntriesCount()); ++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); ++ assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); ++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); ++ } ++ } ++ ++ @Test ++ public void testAckMessageInBatch() throws Throwable { ++ ConfigurationManager.getProxyConfig().setEnableBatchAck(true); ++ ++ String successMessageId = "msg1"; ++ String notOkMessageId = "msg2"; ++ String exceptionMessageId = "msg3"; ++ ++ doAnswer((Answer>>) invocation -> { ++ List receiptHandleMessageList = invocation.getArgument(1, List.class); ++ List batchAckResultList = new ArrayList<>(); ++ for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) { ++ BatchAckResult batchAckResult; ++ if (receiptHandleMessage.getMessageId().equals(successMessageId)) { ++ AckResult ackResult = new AckResult(); ++ ackResult.setStatus(AckStatus.OK); ++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); ++ } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) { ++ AckResult ackResult = new AckResult(); ++ ackResult.setStatus(AckStatus.NO_EXIST); ++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); ++ } else { ++ batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "")); ++ } ++ batchAckResultList.add(batchAckResult); ++ } ++ return CompletableFuture.completedFuture(batchAckResultList); ++ }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString()); ++ ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(successMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.OK, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(notOkMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(exceptionMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); ++ } ++ { ++ AckMessageResponse response = this.ackMessageActivity.ackMessage( ++ createContext(), ++ AckMessageRequest.newBuilder() ++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++ .setGroup(Resource.newBuilder().setName(GROUP).build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(successMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(notOkMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .addEntries(AckMessageEntry.newBuilder() ++ .setMessageId(exceptionMessageId) ++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++ .build()) ++ .build() ++ ).get(); ++ ++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); ++ assertEquals(3, response.getEntriesCount()); ++ Map msgCode = new HashMap<>(); ++ for (AckMessageResultEntry entry : response.getEntriesList()) { ++ msgCode.put(entry.getMessageId(), entry.getStatus().getCode()); ++ } ++ assertEquals(Code.OK, msgCode.get(successMessageId)); ++ assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId)); ++ assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId)); ++ } + } + } +\ No newline at end of file +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java +index 5c1ea9627..072630e39 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java +@@ -66,14 +66,6 @@ public class BaseProcessorTest extends InitConfigTest { + protected ProxyRelayService proxyRelayService; + @Mock + protected MetadataService metadataService; +- @Mock +- protected ProducerProcessor producerProcessor; +- @Mock +- protected ConsumerProcessor consumerProcessor; +- @Mock +- protected TransactionProcessor transactionProcessor; +- @Mock +- protected ClientProcessor clientProcessor; + + public void before() throws Throwable { + super.before(); +@@ -92,6 +84,13 @@ public class BaseProcessorTest extends InitConfigTest { + } + + protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) { ++ return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(), ++ RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), ++ RANDOM.nextInt(Integer.MAX_VALUE), "mockBroker"); ++ } ++ ++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime, ++ long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) { + MessageExt messageExt = new MessageExt(); + messageExt.setTopic(topic); + messageExt.setTags(tags); +@@ -100,8 +99,7 @@ public class BaseProcessorTest extends InitConfigTest { + messageExt.setMsgId(MessageClientIDSetter.createUniqID()); + messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE)); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, +- ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime, +- RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE))); ++ ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset)); + return messageExt; + } + +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java +index 717e86fc0..db268a06e 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java +@@ -20,8 +20,11 @@ package org.apache.rocketmq.proxy.processor; + import com.google.common.collect.Sets; + import java.time.Duration; + import java.util.ArrayList; ++import java.util.Collections; ++import java.util.HashMap; + import java.util.HashSet; + import java.util.List; ++import java.util.Map; + import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.Executors; +@@ -39,7 +42,10 @@ import org.apache.rocketmq.common.message.MessageClientIDSetter; + import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.proxy.common.ProxyContext; ++import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++import org.apache.rocketmq.proxy.common.utils.FutureUtils; + import org.apache.rocketmq.proxy.common.utils.ProxyUtils; ++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; + import org.apache.rocketmq.proxy.service.route.MessageQueueView; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; +@@ -50,16 +56,22 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; + import org.junit.Before; + import org.junit.Test; + import org.mockito.ArgumentCaptor; ++import org.mockito.stubbing.Answer; + + import static org.assertj.core.api.Assertions.assertThat; + import static org.junit.Assert.assertEquals; + import static org.junit.Assert.assertNotNull; ++import static org.junit.Assert.assertNull; + import static org.junit.Assert.assertSame; + import static org.mockito.ArgumentMatchers.any; ++import static org.mockito.ArgumentMatchers.anyList; + import static org.mockito.ArgumentMatchers.anyLong; + import static org.mockito.ArgumentMatchers.anyString; + import static org.mockito.ArgumentMatchers.eq; ++import static org.mockito.Mockito.doAnswer; + import static org.mockito.Mockito.mock; ++import static org.mockito.Mockito.never; ++import static org.mockito.Mockito.verify; + import static org.mockito.Mockito.when; + + public class ConsumerProcessorTest extends BaseProcessorTest { +@@ -162,6 +174,109 @@ public class ConsumerProcessorTest extends BaseProcessorTest { + assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo()); + } + ++ @Test ++ public void testBatchAckExpireMessage() throws Throwable { ++ String brokerName1 = "brokerName1"; ++ ++ List receiptHandleMessageList = new ArrayList<>(); ++ for (int i = 0; i < 3; i++) { ++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, ++ 0, 0, 0, i, brokerName1); ++ ReceiptHandle expireHandle = create(expireMessage); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); ++ } ++ ++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); ++ ++ verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); ++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); ++ for (BatchAckResult batchAckResult : batchAckResultList) { ++ assertNull(batchAckResult.getAckResult()); ++ assertNotNull(batchAckResult.getProxyException()); ++ assertNotNull(batchAckResult.getReceiptHandleMessage()); ++ } ++ ++ } ++ ++ @Test ++ public void testBatchAckMessage() throws Throwable { ++ String brokerName1 = "brokerName1"; ++ String brokerName2 = "brokerName2"; ++ String errThrowBrokerName = "errThrowBrokerName"; ++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, ++ 0, 0, 0, 0, brokerName1); ++ ReceiptHandle expireHandle = create(expireMessage); ++ ++ List receiptHandleMessageList = new ArrayList<>(); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); ++ List broker1Msg = new ArrayList<>(); ++ List broker2Msg = new ArrayList<>(); ++ ++ long now = System.currentTimeMillis(); ++ int msgNum = 3; ++ for (int i = 0; i < msgNum; i++) { ++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, ++ 0, 0, 0, i + 1, brokerName1); ++ ReceiptHandle brokerHandle = create(brokerMessage); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); ++ broker1Msg.add(brokerMessage.getMsgId()); ++ } ++ for (int i = 0; i < msgNum; i++) { ++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, ++ 0, 0, 0, i + 1, brokerName2); ++ ReceiptHandle brokerHandle = create(brokerMessage); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); ++ broker2Msg.add(brokerMessage.getMsgId()); ++ } ++ ++ // for this message, will throw exception in batchAckMessage ++ MessageExt errThrowMessage = createMessageExt(TOPIC, "", 0, 3000, now, ++ 0, 0, 0, 0, errThrowBrokerName); ++ ReceiptHandle errThrowHandle = create(errThrowMessage); ++ receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId())); ++ ++ Collections.shuffle(receiptHandleMessageList); ++ ++ doAnswer((Answer>) invocation -> { ++ List handleMessageList = invocation.getArgument(1, List.class); ++ AckResult ackResult = new AckResult(); ++ String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName(); ++ if (brokerName.equals(brokerName1)) { ++ ackResult.setStatus(AckStatus.OK); ++ } else if (brokerName.equals(brokerName2)) { ++ ackResult.setStatus(AckStatus.NO_EXIST); ++ } else { ++ return FutureUtils.completeExceptionally(new RuntimeException()); ++ } ++ ++ return CompletableFuture.completedFuture(ackResult); ++ }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); ++ ++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); ++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); ++ ++ // check ackResult for each msg ++ Map msgBatchAckResult = new HashMap<>(); ++ for (BatchAckResult batchAckResult : batchAckResultList) { ++ msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult); ++ } ++ for (String msgId : broker1Msg) { ++ assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus()); ++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); ++ } ++ for (String msgId : broker2Msg) { ++ assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus()); ++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); ++ } ++ assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException()); ++ assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode()); ++ assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult()); ++ ++ assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException()); ++ assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode()); ++ assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult()); ++ } ++ + @Test + public void testChangeInvisibleTime() throws Throwable { + ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000)); +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +index 77a119a29..3f3a4ae40 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +@@ -220,6 +220,18 @@ public class MQClientAPIExtTest { + assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get()); + } + ++ @Test ++ public void testBatchAckMessageAsync() throws Exception { ++ AckResult ackResult = new AckResult(); ++ doAnswer((Answer) mock -> { ++ AckCallback ackCallback = mock.getArgument(2); ++ ackCallback.onSuccess(ackResult); ++ return null; ++ }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any()); ++ ++ assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get()); ++ } ++ + @Test + public void testChangeInvisibleTimeAsync() throws Exception { + AckResult ackResult = new AckResult(); +-- +2.32.0.windows.2 + diff --git a/patch013-backport-enhance-admin-output.patch b/patch013-backport-enhance-admin-output.patch new file mode 100644 index 000000000..3fa60916f --- /dev/null +++ b/patch013-backport-enhance-admin-output.patch @@ -0,0 +1,892 @@ +From 7e018520ef707a841c66c55d621f6560d03b631b Mon Sep 17 00:00:00 2001 +From: Zhouxiang Zhan +Date: Fri, 25 Aug 2023 09:49:22 +0800 +Subject: [PATCH 1/6] Add expireAfterAccess for cache (#7247) + +Add expireAfterAccess for cache +--- + .../rocketmq/proxy/config/ProxyConfig.java | 59 ++++++++++++++----- + .../metadata/ClusterMetadataService.java | 6 +- + .../service/route/TopicRouteService.java | 14 +++-- + 3 files changed, 56 insertions(+), 23 deletions(-) + +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +index 76a243919..2994893d7 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +@@ -155,14 +155,17 @@ public class ProxyConfig implements ConfigFile { + private int consumerProcessorThreadPoolQueueCapacity = 10000; + + private boolean useEndpointPortFromRequest = false; +- private int topicRouteServiceCacheExpiredInSeconds = 20; ++ ++ private int topicRouteServiceCacheExpiredSeconds = 300; ++ private int topicRouteServiceCacheRefreshSeconds = 20; + private int topicRouteServiceCacheMaxNum = 20000; + private int topicRouteServiceThreadPoolNums = PROCESSOR_NUMBER; + private int topicRouteServiceThreadPoolQueueCapacity = 5000; +- +- private int topicConfigCacheExpiredInSeconds = 20; ++ private int topicConfigCacheExpiredSeconds = 300; ++ private int topicConfigCacheRefreshSeconds = 20; + private int topicConfigCacheMaxNum = 20000; +- private int subscriptionGroupConfigCacheExpiredInSeconds = 20; ++ private int subscriptionGroupConfigCacheExpiredSeconds = 300; ++ private int subscriptionGroupConfigCacheRefreshSeconds = 20; + private int subscriptionGroupConfigCacheMaxNum = 20000; + private int metadataThreadPoolNums = 3; + private int metadataThreadPoolQueueCapacity = 100000; +@@ -794,12 +797,20 @@ public class ProxyConfig implements ConfigFile { + this.consumerProcessorThreadPoolQueueCapacity = consumerProcessorThreadPoolQueueCapacity; + } + +- public int getTopicRouteServiceCacheExpiredInSeconds() { +- return topicRouteServiceCacheExpiredInSeconds; ++ public int getTopicRouteServiceCacheExpiredSeconds() { ++ return topicRouteServiceCacheExpiredSeconds; ++ } ++ ++ public void setTopicRouteServiceCacheExpiredSeconds(int topicRouteServiceCacheExpiredSeconds) { ++ this.topicRouteServiceCacheExpiredSeconds = topicRouteServiceCacheExpiredSeconds; + } + +- public void setTopicRouteServiceCacheExpiredInSeconds(int topicRouteServiceCacheExpiredInSeconds) { +- this.topicRouteServiceCacheExpiredInSeconds = topicRouteServiceCacheExpiredInSeconds; ++ public int getTopicRouteServiceCacheRefreshSeconds() { ++ return topicRouteServiceCacheRefreshSeconds; ++ } ++ ++ public void setTopicRouteServiceCacheRefreshSeconds(int topicRouteServiceCacheRefreshSeconds) { ++ this.topicRouteServiceCacheRefreshSeconds = topicRouteServiceCacheRefreshSeconds; + } + + public int getTopicRouteServiceCacheMaxNum() { +@@ -826,12 +837,20 @@ public class ProxyConfig implements ConfigFile { + this.topicRouteServiceThreadPoolQueueCapacity = topicRouteServiceThreadPoolQueueCapacity; + } + +- public int getTopicConfigCacheExpiredInSeconds() { +- return topicConfigCacheExpiredInSeconds; ++ public int getTopicConfigCacheRefreshSeconds() { ++ return topicConfigCacheRefreshSeconds; ++ } ++ ++ public void setTopicConfigCacheRefreshSeconds(int topicConfigCacheRefreshSeconds) { ++ this.topicConfigCacheRefreshSeconds = topicConfigCacheRefreshSeconds; ++ } ++ ++ public int getTopicConfigCacheExpiredSeconds() { ++ return topicConfigCacheExpiredSeconds; + } + +- public void setTopicConfigCacheExpiredInSeconds(int topicConfigCacheExpiredInSeconds) { +- this.topicConfigCacheExpiredInSeconds = topicConfigCacheExpiredInSeconds; ++ public void setTopicConfigCacheExpiredSeconds(int topicConfigCacheExpiredSeconds) { ++ this.topicConfigCacheExpiredSeconds = topicConfigCacheExpiredSeconds; + } + + public int getTopicConfigCacheMaxNum() { +@@ -842,12 +861,20 @@ public class ProxyConfig implements ConfigFile { + this.topicConfigCacheMaxNum = topicConfigCacheMaxNum; + } + +- public int getSubscriptionGroupConfigCacheExpiredInSeconds() { +- return subscriptionGroupConfigCacheExpiredInSeconds; ++ public int getSubscriptionGroupConfigCacheRefreshSeconds() { ++ return subscriptionGroupConfigCacheRefreshSeconds; ++ } ++ ++ public void setSubscriptionGroupConfigCacheRefreshSeconds(int subscriptionGroupConfigCacheRefreshSeconds) { ++ this.subscriptionGroupConfigCacheRefreshSeconds = subscriptionGroupConfigCacheRefreshSeconds; ++ } ++ ++ public int getSubscriptionGroupConfigCacheExpiredSeconds() { ++ return subscriptionGroupConfigCacheExpiredSeconds; + } + +- public void setSubscriptionGroupConfigCacheExpiredInSeconds(int subscriptionGroupConfigCacheExpiredInSeconds) { +- this.subscriptionGroupConfigCacheExpiredInSeconds = subscriptionGroupConfigCacheExpiredInSeconds; ++ public void setSubscriptionGroupConfigCacheExpiredSeconds(int subscriptionGroupConfigCacheExpiredSeconds) { ++ this.subscriptionGroupConfigCacheExpiredSeconds = subscriptionGroupConfigCacheExpiredSeconds; + } + + public int getSubscriptionGroupConfigCacheMaxNum() { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java +index bc9582ad8..d34a0efd9 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java +@@ -69,11 +69,13 @@ public class ClusterMetadataService extends AbstractStartAndShutdown implements + ); + this.topicConfigCache = CacheBuilder.newBuilder() + .maximumSize(config.getTopicConfigCacheMaxNum()) +- .refreshAfterWrite(config.getTopicConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) ++ .expireAfterAccess(config.getTopicConfigCacheExpiredSeconds(), TimeUnit.SECONDS) ++ .refreshAfterWrite(config.getTopicConfigCacheRefreshSeconds(), TimeUnit.SECONDS) + .build(new ClusterTopicConfigCacheLoader()); + this.subscriptionGroupConfigCache = CacheBuilder.newBuilder() + .maximumSize(config.getSubscriptionGroupConfigCacheMaxNum()) +- .refreshAfterWrite(config.getSubscriptionGroupConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) ++ .expireAfterAccess(config.getSubscriptionGroupConfigCacheExpiredSeconds(), TimeUnit.SECONDS) ++ .refreshAfterWrite(config.getSubscriptionGroupConfigCacheRefreshSeconds(), TimeUnit.SECONDS) + .build(new ClusterSubscriptionGroupConfigCacheLoader()); + + this.init(); +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +index e012a5465..84348adc3 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +@@ -68,10 +68,13 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { + ); + this.mqClientAPIFactory = mqClientAPIFactory; + +- this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum()). +- refreshAfterWrite(config.getTopicRouteServiceCacheExpiredInSeconds(), TimeUnit.SECONDS). +- executor(cacheRefreshExecutor).build(new CacheLoader() { +- @Override public @Nullable MessageQueueView load(String topic) throws Exception { ++ this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum()) ++ .expireAfterAccess(config.getTopicRouteServiceCacheExpiredSeconds(), TimeUnit.SECONDS) ++ .refreshAfterWrite(config.getTopicRouteServiceCacheRefreshSeconds(), TimeUnit.SECONDS) ++ .executor(cacheRefreshExecutor) ++ .build(new CacheLoader() { ++ @Override ++ public @Nullable MessageQueueView load(String topic) throws Exception { + try { + TopicRouteData topicRouteData = mqClientAPIFactory.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis()); + return buildMessageQueueView(topic, topicRouteData); +@@ -83,7 +86,8 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { + } + } + +- @Override public @Nullable MessageQueueView reload(@NonNull String key, ++ @Override ++ public @Nullable MessageQueueView reload(@NonNull String key, + @NonNull MessageQueueView oldValue) throws Exception { + try { + return load(key); +-- +2.32.0.windows.2 + + +From 5f6dc90f9dab35809fcb0407d4d5cc2737d2335e Mon Sep 17 00:00:00 2001 +From: Ziyi Tan +Date: Fri, 25 Aug 2023 11:17:23 +0800 +Subject: [PATCH 2/6] [ISSUE #7250] Beautify command rocksDBConfigToJson output + +Co-authored-by: Ziy1-Tan +--- + .../metadata/RocksDBConfigToJsonCommand.java | 32 +++++++++++-------- + 1 file changed, 18 insertions(+), 14 deletions(-) + +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +index 3053f4684..3fc63e4dd 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +@@ -21,13 +21,13 @@ import org.apache.commons.cli.CommandLine; + import org.apache.commons.cli.Option; + import org.apache.commons.cli.Options; + import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.UtilAll; + import org.apache.rocketmq.common.config.RocksDBConfigManager; + import org.apache.rocketmq.common.utils.DataConverter; + import org.apache.rocketmq.remoting.RPCHook; + import org.apache.rocketmq.tools.command.SubCommand; + import org.apache.rocketmq.tools.command.SubCommandException; + +-import java.io.File; + import java.util.HashMap; + import java.util.Map; + +@@ -48,7 +48,7 @@ public class RocksDBConfigToJsonCommand implements SubCommand { + @Override + public Options buildCommandlineOptions(Options options) { + Option pathOption = new Option("p", "path", true, +- "Absolute path to the metadata directory"); ++ "Absolute path for the metadata directory"); + pathOption.setRequired(true); + options.addOption(pathOption); + +@@ -63,15 +63,14 @@ public class RocksDBConfigToJsonCommand implements SubCommand { + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { + String path = commandLine.getOptionValue("path").trim(); +- if (StringUtils.isEmpty(path) || !new File(path).exists()) { ++ if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { + System.out.print("Rocksdb path is invalid.\n"); + return; + } + + String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); + +- final long memTableFlushInterval = 60 * 60 * 1000L; +- RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); ++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(60 * 60 * 1000L); + try { + if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { + // for topics.json +@@ -84,13 +83,16 @@ public class RocksDBConfigToJsonCommand implements SubCommand { + topicConfigTable.put(topic, jsonObject); + }); + +- if (isLoad) { +- topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); +- final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); +- System.out.print(topicsJsonStr + "\n"); ++ if (!isLoad) { ++ System.out.print("RocksDB load error, path=" + path); + return; + } ++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); ++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); ++ System.out.print(topicsJsonStr + "\n"); ++ return; + } ++ + if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { + // for subscriptionGroup.json + final Map subscriptionGroupJsonConfig = new HashMap<>(); +@@ -102,13 +104,15 @@ public class RocksDBConfigToJsonCommand implements SubCommand { + subscriptionGroupTable.put(subscriptionGroup, jsonObject); + }); + +- if (isLoad) { +- subscriptionGroupJsonConfig.put("subscriptionGroupTable", +- (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); +- final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); +- System.out.print(subscriptionGroupJsonStr + "\n"); ++ if (!isLoad) { ++ System.out.print("RocksDB load error, path=" + path); + return; + } ++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", ++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); ++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); ++ System.out.print(subscriptionGroupJsonStr + "\n"); ++ return; + } + System.out.print("Config type was not recognized, configType=" + configType + "\n"); + } finally { +-- +2.32.0.windows.2 + + +From b4f73e2aabc1b141cec98431899e4090340adf0f Mon Sep 17 00:00:00 2001 +From: mxsm +Date: Sun, 27 Aug 2023 20:58:58 +0800 +Subject: [PATCH 3/6] [ISSUE #7271] Optimize the configuration for setting the + quantity of TimerDequeuePutMessageService (#7272) + +--- + .../java/org/apache/rocketmq/store/timer/TimerMessageStore.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +index 690f4863e..181f7087a 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +@@ -222,7 +222,7 @@ public class TimerMessageStore { + dequeueGetMessageServices[i] = new TimerDequeueGetMessageService(); + } + +- int putThreadNum = Math.max(storeConfig.getTimerGetMessageThreadNum(), 1); ++ int putThreadNum = Math.max(storeConfig.getTimerPutMessageThreadNum(), 1); + dequeuePutMessageServices = new TimerDequeuePutMessageService[putThreadNum]; + for (int i = 0; i < dequeuePutMessageServices.length; i++) { + dequeuePutMessageServices[i] = new TimerDequeuePutMessageService(); +-- +2.32.0.windows.2 + + +From 3e100103af68588528bf32f3752a85e8023f46f8 Mon Sep 17 00:00:00 2001 +From: Ziyi Tan +Date: Tue, 29 Aug 2023 13:48:51 +0800 +Subject: [PATCH 4/6] [ISSUE #7277] Enhance rocksDBConfigToJson to support + metadata counting (#7276) + +--- + .../common/config/AbstractRocksDBStorage.java | 4 +- + .../common/config/ConfigRocksDBStorage.java | 6 + + .../tools/command/MQAdminStartup.java | 4 +- + .../ExportMetadataInRocksDBCommand.java | 138 ++++++++++++++++++ + .../metadata/RocksDBConfigToJsonCommand.java | 122 ---------------- + ...> ExportMetadataInRocksDBCommandTest.java} | 38 +++-- + 6 files changed, 173 insertions(+), 139 deletions(-) + create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java + delete mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java + rename tools/src/test/java/org/apache/rocketmq/tools/command/metadata/{KvConfigToJsonCommandTest.java => ExportMetadataInRocksDBCommandTest.java} (62%) + +diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +index e3673baad..a720a5be3 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +@@ -385,8 +385,10 @@ public abstract class AbstractRocksDBStorage { + this.options.close(); + } + //4. close db. +- if (db != null) { ++ if (db != null && !this.readOnly) { + this.db.syncWal(); ++ } ++ if (db != null) { + this.db.closeE(); + } + //5. help gc. +diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +index 9d05ed282..463bd8fed 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java ++++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +@@ -60,6 +60,12 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage { + this.readOnly = false; + } + ++ public ConfigRocksDBStorage(final String dbPath, boolean readOnly) { ++ super(); ++ this.dbPath = dbPath; ++ this.readOnly = readOnly; ++ } ++ + private void initOptions() { + this.options = createConfigDBOptions(); + +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +index 324aa1856..788fa83c2 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +@@ -80,7 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; + import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; + import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; + import org.apache.rocketmq.tools.command.message.SendMessageCommand; +-import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; ++import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand; + import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; + import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; + import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; +@@ -212,7 +212,6 @@ public class MQAdminStartup { + + initCommand(new ClusterListSubCommand()); + initCommand(new TopicListSubCommand()); +- initCommand(new RocksDBConfigToJsonCommand()); + + initCommand(new UpdateKvConfigCommand()); + initCommand(new DeleteKvConfigCommand()); +@@ -257,6 +256,7 @@ public class MQAdminStartup { + initCommand(new ExportMetadataCommand()); + initCommand(new ExportConfigsCommand()); + initCommand(new ExportMetricsCommand()); ++ initCommand(new ExportMetadataInRocksDBCommand()); + + initCommand(new HAStatusSubCommand()); + +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +new file mode 100644 +index 000000000..2a7d3fba4 +--- /dev/null ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +@@ -0,0 +1,138 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.tools.command.export; ++ ++import com.alibaba.fastjson.JSONObject; ++import org.apache.commons.cli.CommandLine; ++import org.apache.commons.cli.Option; ++import org.apache.commons.cli.Options; ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.UtilAll; ++import org.apache.rocketmq.common.config.ConfigRocksDBStorage; ++import org.apache.rocketmq.common.utils.DataConverter; ++import org.apache.rocketmq.remoting.RPCHook; ++import org.apache.rocketmq.tools.command.SubCommand; ++import org.apache.rocketmq.tools.command.SubCommandException; ++import org.rocksdb.RocksIterator; ++ ++import java.util.HashMap; ++import java.util.Map; ++import java.util.concurrent.atomic.AtomicLong; ++import java.util.function.BiConsumer; ++ ++public class ExportMetadataInRocksDBCommand implements SubCommand { ++ private static final String TOPICS_JSON_CONFIG = "topics"; ++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; ++ ++ @Override ++ public String commandName() { ++ return "exportMetadataInRocksDB"; ++ } ++ ++ @Override ++ public String commandDesc() { ++ return "export RocksDB kv config (topics/subscriptionGroups)"; ++ } ++ ++ @Override ++ public Options buildCommandlineOptions(Options options) { ++ Option pathOption = new Option("p", "path", true, ++ "Absolute path for the metadata directory"); ++ pathOption.setRequired(true); ++ options.addOption(pathOption); ++ ++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + ++ "topics/subscriptionGroups"); ++ configTypeOption.setRequired(true); ++ options.addOption(configTypeOption); ++ ++ Option jsonEnableOption = new Option("j", "jsonEnable", true, ++ "Json format enable, Default: false"); ++ jsonEnableOption.setRequired(false); ++ options.addOption(jsonEnableOption); ++ ++ return options; ++ } ++ ++ @Override ++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { ++ String path = commandLine.getOptionValue("path").trim(); ++ if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { ++ System.out.print("RocksDB path is invalid.\n"); ++ return; ++ } ++ ++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); ++ ++ boolean jsonEnable = false; ++ if (commandLine.hasOption("jsonEnable")) { ++ jsonEnable = Boolean.parseBoolean(commandLine.getOptionValue("jsonEnable").trim()); ++ } ++ ++ ++ ConfigRocksDBStorage kvStore = new ConfigRocksDBStorage(path, true /* readOnly */); ++ if (!kvStore.start()) { ++ System.out.print("RocksDB load error, path=" + path + "\n"); ++ return; ++ } ++ ++ try { ++ if (TOPICS_JSON_CONFIG.equalsIgnoreCase(configType) || SUBSCRIPTION_GROUP_JSON_CONFIG.equalsIgnoreCase(configType)) { ++ handleExportMetadata(kvStore, configType, jsonEnable); ++ } else { ++ System.out.printf("Invalid config type=%s, Options: topics,subscriptionGroups\n", configType); ++ } ++ } finally { ++ kvStore.shutdown(); ++ } ++ } ++ ++ private static void handleExportMetadata(ConfigRocksDBStorage kvStore, String configType, boolean jsonEnable) { ++ if (jsonEnable) { ++ final Map jsonConfig = new HashMap<>(); ++ final Map configTable = new HashMap<>(); ++ iterateKvStore(kvStore, (key, value) -> { ++ final String configKey = new String(key, DataConverter.charset); ++ final String configValue = new String(value, DataConverter.charset); ++ final JSONObject jsonObject = JSONObject.parseObject(configValue); ++ configTable.put(configKey, jsonObject); ++ } ++ ); ++ ++ jsonConfig.put(configType.equalsIgnoreCase(TOPICS_JSON_CONFIG) ? "topicConfigTable" : "subscriptionGroupTable", ++ (JSONObject) JSONObject.toJSON(configTable)); ++ final String jsonConfigStr = JSONObject.toJSONString(jsonConfig, true); ++ System.out.print(jsonConfigStr + "\n"); ++ } else { ++ AtomicLong count = new AtomicLong(0); ++ iterateKvStore(kvStore, (key, value) -> { ++ final String configKey = new String(key, DataConverter.charset); ++ final String configValue = new String(value, DataConverter.charset); ++ System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), configKey, configValue); ++ }); ++ } ++ } ++ ++ private static void iterateKvStore(ConfigRocksDBStorage kvStore, BiConsumer biConsumer) { ++ try (RocksIterator iterator = kvStore.iterator()) { ++ iterator.seekToFirst(); ++ for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { ++ biConsumer.accept(iterator.key(), iterator.value()); ++ } ++ } ++ } ++} +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +deleted file mode 100644 +index 3fc63e4dd..000000000 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++++ /dev/null +@@ -1,122 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package org.apache.rocketmq.tools.command.metadata; +- +-import com.alibaba.fastjson.JSONObject; +-import org.apache.commons.cli.CommandLine; +-import org.apache.commons.cli.Option; +-import org.apache.commons.cli.Options; +-import org.apache.commons.lang3.StringUtils; +-import org.apache.rocketmq.common.UtilAll; +-import org.apache.rocketmq.common.config.RocksDBConfigManager; +-import org.apache.rocketmq.common.utils.DataConverter; +-import org.apache.rocketmq.remoting.RPCHook; +-import org.apache.rocketmq.tools.command.SubCommand; +-import org.apache.rocketmq.tools.command.SubCommandException; +- +-import java.util.HashMap; +-import java.util.Map; +- +-public class RocksDBConfigToJsonCommand implements SubCommand { +- private static final String TOPICS_JSON_CONFIG = "topics"; +- private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; +- +- @Override +- public String commandName() { +- return "rocksDBConfigToJson"; +- } +- +- @Override +- public String commandDesc() { +- return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; +- } +- +- @Override +- public Options buildCommandlineOptions(Options options) { +- Option pathOption = new Option("p", "path", true, +- "Absolute path for the metadata directory"); +- pathOption.setRequired(true); +- options.addOption(pathOption); +- +- Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + +- "topics/subscriptionGroups"); +- configTypeOption.setRequired(true); +- options.addOption(configTypeOption); +- +- return options; +- } +- +- @Override +- public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { +- String path = commandLine.getOptionValue("path").trim(); +- if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { +- System.out.print("Rocksdb path is invalid.\n"); +- return; +- } +- +- String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); +- +- RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(60 * 60 * 1000L); +- try { +- if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { +- // for topics.json +- final Map topicsJsonConfig = new HashMap<>(); +- final Map topicConfigTable = new HashMap<>(); +- boolean isLoad = kvConfigManager.load(path, (key, value) -> { +- final String topic = new String(key, DataConverter.charset); +- final String topicConfig = new String(value, DataConverter.charset); +- final JSONObject jsonObject = JSONObject.parseObject(topicConfig); +- topicConfigTable.put(topic, jsonObject); +- }); +- +- if (!isLoad) { +- System.out.print("RocksDB load error, path=" + path); +- return; +- } +- topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); +- final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); +- System.out.print(topicsJsonStr + "\n"); +- return; +- } +- +- if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { +- // for subscriptionGroup.json +- final Map subscriptionGroupJsonConfig = new HashMap<>(); +- final Map subscriptionGroupTable = new HashMap<>(); +- boolean isLoad = kvConfigManager.load(path, (key, value) -> { +- final String subscriptionGroup = new String(key, DataConverter.charset); +- final String subscriptionGroupConfig = new String(value, DataConverter.charset); +- final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); +- subscriptionGroupTable.put(subscriptionGroup, jsonObject); +- }); +- +- if (!isLoad) { +- System.out.print("RocksDB load error, path=" + path); +- return; +- } +- subscriptionGroupJsonConfig.put("subscriptionGroupTable", +- (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); +- final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); +- System.out.print(subscriptionGroupJsonStr + "\n"); +- return; +- } +- System.out.print("Config type was not recognized, configType=" + configType + "\n"); +- } finally { +- kvConfigManager.stop(); +- } +- } +-} +diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java +similarity index 62% +rename from tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java +rename to tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java +index b2f66c7b0..2b938c90f 100644 +--- a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java +@@ -21,43 +21,53 @@ import org.apache.commons.cli.DefaultParser; + import org.apache.commons.cli.Options; + import org.apache.rocketmq.srvutil.ServerUtil; + import org.apache.rocketmq.tools.command.SubCommandException; ++import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand; + import org.junit.Test; + + import java.io.File; + + import static org.assertj.core.api.Assertions.assertThat; + +-public class KvConfigToJsonCommandTest { ++public class ExportMetadataInRocksDBCommandTest { + private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; + + @Test + public void testExecute() throws SubCommandException { + { +- String[] cases = new String[]{"topics", "subscriptionGroups"}; +- for (String c : cases) { +- RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); ++ String[][] cases = new String[][] { ++ {"topics", "false"}, ++ {"topics", "false1"}, ++ {"topics", "true"}, ++ {"subscriptionGroups", "false"}, ++ {"subscriptionGroups", "false2"}, ++ {"subscriptionGroups", "true"} ++ }; ++ ++ for (String[] c : cases) { ++ ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); +- String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c}; ++ String[] subargs = new String[] {"-p " + BASE_PATH + c[0], "-t " + c[0], "-j " + c[1]}; + final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, +- cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.buildCommandlineOptions(options), new DefaultParser()); + cmd.execute(commandLine, options, null); +- assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c); +- assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c); ++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c[0]); ++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c[0]); ++ assertThat(commandLine.getOptionValue("j").trim()).isEqualTo(c[1]); + } + } + // invalid cases + { +- String[][] cases = new String[][]{ +- {"-p " + BASE_PATH + "tmpPath", "-t topics"}, +- {"-p ", "-t topics"}, +- {"-p " + BASE_PATH + "topics", "-t invalid_type"} ++ String[][] cases = new String[][] { ++ {"-p " + BASE_PATH + "tmpPath", "-t topics", "-j true"}, ++ {"-p ", "-t topics", "-j true"}, ++ {"-p " + BASE_PATH + "topics", "-t invalid_type", "-j true"} + }; + + for (String[] c : cases) { +- RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); ++ ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, +- cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.buildCommandlineOptions(options), new DefaultParser()); + cmd.execute(commandLine, options, null); + } + } +-- +2.32.0.windows.2 + + +From fa549154370cb866a90e37c13a90d2c598d6b1f6 Mon Sep 17 00:00:00 2001 +From: yuz10 <845238369@qq.com> +Date: Tue, 29 Aug 2023 15:22:09 +0800 +Subject: [PATCH 5/6] [ISSUE #7261] Slave high CPU usage when + enableScheduleAsyncDeliver=true (#7262) + +* [ISSUE #6390] Add break to the exception of WHEEL_TIMER_NOT_ENABLE. + +* fix broker start fail if mapped file size is 0 + +* log + +* only delete the last empty file + +* change dataReadAheadEnable default to true + +* fix endless loop when master change to slave. +--- + .../rocketmq/broker/schedule/ScheduleMessageService.java | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +index aed0ee19f..297b14207 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +@@ -566,7 +566,8 @@ public class ScheduleMessageService extends ConfigManager { + pendingQueue.remove(); + break; + case RUNNING: +- break; ++ scheduleNextTask(); ++ return; + case EXCEPTION: + if (!isStarted()) { + log.warn("HandlePutResultTask shutdown, info={}", putResultProcess.toString()); +@@ -586,6 +587,10 @@ public class ScheduleMessageService extends ConfigManager { + } + } + ++ scheduleNextTask(); ++ } ++ ++ private void scheduleNextTask() { + if (isStarted()) { + ScheduleMessageService.this.handleExecutorService + .schedule(new HandlePutResultTask(this.delayLevel), DELAY_FOR_A_SLEEP, TimeUnit.MILLISECONDS); +-- +2.32.0.windows.2 + + +From 9f34f55e1dac495730c9cd5469f2ab3225b8f0b9 Mon Sep 17 00:00:00 2001 +From: ShuangxiDing +Date: Tue, 29 Aug 2023 15:48:46 +0800 +Subject: [PATCH 6/6] [ISSUE #7226] Filter tlvs in ppv2 which contents not are + spec-compliant ASCII characters and space (#7227) + +Filter tlvs in ppv2 which not are spec-compliant ASCII characters and space +--- + .../rocketmq/common/utils/BinaryUtil.java | 17 +++++++++++++++++ + .../grpc/ProxyAndTlsProtocolNegotiator.java | 8 +++++++- + .../remoting/netty/NettyRemotingServer.java | 8 +++++++- + 3 files changed, 31 insertions(+), 2 deletions(-) + +diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java +index 421adaca4..7b4b24819 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java ++++ b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java +@@ -43,4 +43,21 @@ public class BinaryUtil { + byte[] bytes = calculateMd5(content); + return Hex.encodeHexString(bytes, false); + } ++ ++ /** ++ * Returns true if subject contains only bytes that are spec-compliant ASCII characters. ++ * @param subject ++ * @return ++ */ ++ public static boolean isAscii(byte[] subject) { ++ if (subject == null) { ++ return false; ++ } ++ for (byte b : subject) { ++ if ((b & 0x80) != 0) { ++ return false; ++ } ++ } ++ return true; ++ } + } +\ No newline at end of file +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java +index ee167bd7b..b584ddfbd 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java +@@ -24,6 +24,7 @@ import io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiator; + import io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiators; + import io.grpc.netty.shaded.io.grpc.netty.ProtocolNegotiationEvent; + import io.grpc.netty.shaded.io.netty.buffer.ByteBuf; ++import io.grpc.netty.shaded.io.netty.buffer.ByteBufUtil; + import io.grpc.netty.shaded.io.netty.channel.ChannelHandler; + import io.grpc.netty.shaded.io.netty.channel.ChannelHandlerContext; + import io.grpc.netty.shaded.io.netty.channel.ChannelInboundHandlerAdapter; +@@ -44,6 +45,7 @@ import org.apache.commons.collections.CollectionUtils; + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.common.constant.HAProxyConstants; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.BinaryUtil; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.proxy.config.ConfigurationManager; +@@ -191,9 +193,13 @@ public class ProxyAndTlsProtocolNegotiator implements InternalProtocolNegotiator + } + if (CollectionUtils.isNotEmpty(msg.tlvs())) { + msg.tlvs().forEach(tlv -> { ++ byte[] valueBytes = ByteBufUtil.getBytes(tlv.content()); ++ if (!BinaryUtil.isAscii(valueBytes)) { ++ return; ++ } + Attributes.Key key = AttributeKeys.valueOf( + HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); +- String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8)); ++ String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8)); + builder.set(key, value); + }); + } +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +index 17f138f86..e626260c9 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +@@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty; + + import io.netty.bootstrap.ServerBootstrap; + import io.netty.buffer.ByteBuf; ++import io.netty.buffer.ByteBufUtil; + import io.netty.buffer.PooledByteBufAllocator; + import io.netty.channel.Channel; + import io.netty.channel.ChannelDuplexHandler; +@@ -58,6 +59,7 @@ import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.HAProxyConstants; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.BinaryUtil; + import org.apache.rocketmq.common.utils.NetworkUtil; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +@@ -787,9 +789,13 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti + } + if (CollectionUtils.isNotEmpty(msg.tlvs())) { + msg.tlvs().forEach(tlv -> { ++ byte[] valueBytes = ByteBufUtil.getBytes(tlv.content()); ++ if (!BinaryUtil.isAscii(valueBytes)) { ++ return; ++ } + AttributeKey key = AttributeKeys.valueOf( + HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); +- String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8)); ++ String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8)); + channel.attr(key).set(value); + }); + } +-- +2.32.0.windows.2 + diff --git a/patch014-backport-Queue-Selection-Strategy-Optimization.patch b/patch014-backport-Queue-Selection-Strategy-Optimization.patch new file mode 100644 index 000000000..90d3553aa --- /dev/null +++ b/patch014-backport-Queue-Selection-Strategy-Optimization.patch @@ -0,0 +1,2023 @@ +From b028277018946868838a82a08211071bc231a175 Mon Sep 17 00:00:00 2001 +From: Ji Juntao +Date: Tue, 29 Aug 2023 16:13:38 +0800 +Subject: [PATCH] [ISSUE #6567] [RIP-63] Queue Selection Strategy Optimization + (#6568) + +Optimize the proxy's and client's selection strategy for brokers when sending messages, and use multiple selection strategies as a pipeline to filter suitable queues. +--- + .../apache/rocketmq/client/ClientConfig.java | 54 +++++ + .../client/common/ThreadLocalIndex.java | 8 + + .../rocketmq/client/impl/MQClientAPIImpl.java | 12 +- + .../client/impl/factory/MQClientInstance.java | 7 + + .../impl/producer/DefaultMQProducerImpl.java | 87 ++++++-- + .../impl/producer/TopicPublishInfo.java | 40 ++++ + .../client/latency/LatencyFaultTolerance.java | 66 +++++- + .../latency/LatencyFaultToleranceImpl.java | 189 ++++++++++++++---- + .../client/latency/MQFaultStrategy.java | 155 ++++++++++---- + .../rocketmq/client/latency/Resolver.java | 17 +- + .../client/latency/ServiceDetector.java | 30 +++ + .../LatencyFaultToleranceImplTest.java | 36 +++- + .../processor/DefaultRequestProcessor.java | 24 --- + .../rocketmq/proxy/config/ProxyConfig.java | 46 +++++ + .../grpc/v2/producer/SendMessageActivity.java | 2 +- + .../proxy/processor/ProducerProcessor.java | 18 +- + .../service/route/LocalTopicRouteService.java | 2 +- + .../service/route/MessageQueueSelector.java | 95 ++++++++- + .../proxy/service/route/MessageQueueView.java | 18 +- + .../service/route/TopicRouteService.java | 80 +++++++- + .../consumer/ReceiveMessageActivityTest.java | 5 +- + .../v2/producer/SendMessageActivityTest.java | 82 +++++++- + .../proxy/service/BaseServiceTest.java | 4 +- + .../route/MessageQueueSelectorTest.java | 8 +- + .../sysmessage/HeartbeatSyncerTest.java | 2 +- + .../ClusterTransactionServiceTest.java | 8 +- + 26 files changed, 919 insertions(+), 176 deletions(-) + rename remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java => client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java (65%) + create mode 100644 client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java + +diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +index f87450f66..bb0fe3522 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +@@ -38,6 +38,8 @@ public class ClientConfig { + public static final String SOCKS_PROXY_CONFIG = "com.rocketmq.socks.proxy.config"; + public static final String DECODE_READ_BODY = "com.rocketmq.read.body"; + public static final String DECODE_DECOMPRESS_BODY = "com.rocketmq.decompress.body"; ++ public static final String SEND_LATENCY_ENABLE = "com.rocketmq.sendLatencyEnable"; ++ public static final String START_DETECTOR_ENABLE = "com.rocketmq.startDetectorEnable"; + public static final String HEART_BEAT_V2 = "com.rocketmq.heartbeat.v2"; + private String namesrvAddr = NameServerAddressUtils.getNameServerAddresses(); + private String clientIP = NetworkUtil.getLocalAddress(); +@@ -72,6 +74,8 @@ public class ClientConfig { + private String socksProxyConfig = System.getProperty(SOCKS_PROXY_CONFIG, "{}"); + + private int mqClientApiTimeout = 3 * 1000; ++ private int detectTimeout = 200; ++ private int detectInterval = 2 * 1000; + + private LanguageCode language = LanguageCode.JAVA; + +@@ -81,6 +85,15 @@ public class ClientConfig { + */ + protected boolean enableStreamRequestType = false; + ++ /** ++ * Enable the fault tolerance mechanism of the client sending process. ++ * DO NOT OPEN when ORDER messages are required. ++ * Turning on will interfere with the queue selection functionality, ++ * possibly conflicting with the order message. ++ */ ++ private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false")); ++ private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false")); ++ + public String buildMQClientId() { + StringBuilder sb = new StringBuilder(); + sb.append(this.getClientIP()); +@@ -186,6 +199,10 @@ public class ClientConfig { + this.decodeDecompressBody = cc.decodeDecompressBody; + this.enableStreamRequestType = cc.enableStreamRequestType; + this.useHeartbeatV2 = cc.useHeartbeatV2; ++ this.startDetectorEnable = cc.startDetectorEnable; ++ this.sendLatencyEnable = cc.sendLatencyEnable; ++ this.detectInterval = cc.detectInterval; ++ this.detectTimeout = cc.detectTimeout; + } + + public ClientConfig cloneClientConfig() { +@@ -210,6 +227,10 @@ public class ClientConfig { + cc.decodeDecompressBody = decodeDecompressBody; + cc.enableStreamRequestType = enableStreamRequestType; + cc.useHeartbeatV2 = useHeartbeatV2; ++ cc.startDetectorEnable = startDetectorEnable; ++ cc.sendLatencyEnable = sendLatencyEnable; ++ cc.detectInterval = detectInterval; ++ cc.detectTimeout = detectTimeout; + return cc; + } + +@@ -381,6 +402,38 @@ public class ClientConfig { + this.enableStreamRequestType = enableStreamRequestType; + } + ++ public boolean isSendLatencyEnable() { ++ return sendLatencyEnable; ++ } ++ ++ public void setSendLatencyEnable(boolean sendLatencyEnable) { ++ this.sendLatencyEnable = sendLatencyEnable; ++ } ++ ++ public boolean isStartDetectorEnable() { ++ return startDetectorEnable; ++ } ++ ++ public void setStartDetectorEnable(boolean startDetectorEnable) { ++ this.startDetectorEnable = startDetectorEnable; ++ } ++ ++ public int getDetectTimeout() { ++ return this.detectTimeout; ++ } ++ ++ public void setDetectTimeout(int detectTimeout) { ++ this.detectTimeout = detectTimeout; ++ } ++ ++ public int getDetectInterval() { ++ return this.detectInterval; ++ } ++ ++ public void setDetectInterval(int detectInterval) { ++ this.detectInterval = detectInterval; ++ } ++ + public boolean isUseHeartbeatV2() { + return useHeartbeatV2; + } +@@ -403,6 +456,7 @@ public class ClientConfig { + + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name() + + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout + + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody ++ + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable + + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]"; + } + } +diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java +index 4a3d90135..3a086c13d 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java ++++ b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java +@@ -33,6 +33,14 @@ public class ThreadLocalIndex { + return index & POSITIVE_MASK; + } + ++ public void reset() { ++ int index = Math.abs(random.nextInt(Integer.MAX_VALUE)); ++ if (index < 0) { ++ index = 0; ++ } ++ this.threadLocalIndex.set(index); ++ } ++ + @Override + public String toString() { + return "ThreadLocalIndex{" + +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +index 213c26fd6..3201a493f 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +@@ -666,7 +666,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + } catch (Throwable e) { + } + +- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); ++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); + return; + } + +@@ -684,14 +684,14 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + } catch (Throwable e) { + } + +- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); ++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); + } catch (Exception e) { +- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); ++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, e, context, false, producer); + } + } else { +- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); ++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); + if (!responseFuture.isSendRequestOK()) { + MQClientException ex = new MQClientException("send request failed", responseFuture.getCause()); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +@@ -711,7 +711,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + }); + } catch (Exception ex) { + long cost = System.currentTimeMillis() - beginStartTime; +- producer.updateFaultItem(brokerName, cost, true); ++ producer.updateFaultItem(brokerName, cost, true, false); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); + } +@@ -735,7 +735,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + if (needRetry && tmp <= timesTotal) { + String retryBrokerName = brokerName;//by default, it will send to the same broker + if (topicPublishInfo != null) { //select one message queue accordingly, in order to determine which broker to send +- MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName); ++ MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName, false); + retryBrokerName = instance.getBrokerNameFromMessageQueue(mqChosen); + } + String addr = instance.findBrokerAddressInPublish(retryBrokerName); +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +index 8851bc815..9484b26f8 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +@@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.ThreadFactory; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicLong; + import java.util.concurrent.locks.Lock; +@@ -125,6 +126,12 @@ public class MQClientInstance { + private final Set brokerSupportV2HeartbeatSet = new HashSet(); + private final ConcurrentMap brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap(); + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "MQClientFactoryScheduledThread")); ++ private final ScheduledExecutorService fetchRemoteConfigExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { ++ @Override ++ public Thread newThread(Runnable r) { ++ return new Thread(r, "MQClientFactoryFetchRemoteConfigScheduledThread"); ++ } ++ }); + private final PullMessageService pullMessageService; + private final RebalanceService rebalanceService; + private final DefaultMQProducer defaultMQProducer; +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +index 3f4c6e5f7..bbbb17b07 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +@@ -33,6 +33,8 @@ import java.util.concurrent.RejectedExecutionException; + import java.util.concurrent.Semaphore; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; ++ ++import com.google.common.base.Optional; + import org.apache.rocketmq.client.QueryResult; + import org.apache.rocketmq.client.Validators; + import org.apache.rocketmq.client.common.ClientErrorCode; +@@ -49,6 +51,8 @@ import org.apache.rocketmq.client.impl.CommunicationMode; + import org.apache.rocketmq.client.impl.MQClientManager; + import org.apache.rocketmq.client.impl.factory.MQClientInstance; + import org.apache.rocketmq.client.latency.MQFaultStrategy; ++import org.apache.rocketmq.client.latency.Resolver; ++import org.apache.rocketmq.client.latency.ServiceDetector; + import org.apache.rocketmq.client.producer.DefaultMQProducer; + import org.apache.rocketmq.client.producer.LocalTransactionExecuter; + import org.apache.rocketmq.client.producer.LocalTransactionState; +@@ -112,7 +116,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mQClientFactory; + private ArrayList checkForbiddenHookList = new ArrayList<>(); +- private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); ++ private MQFaultStrategy mqFaultStrategy; + private ExecutorService asyncSenderExecutor; + + // compression related +@@ -153,8 +157,38 @@ public class DefaultMQProducerImpl implements MQProducerInner { + semaphoreAsyncSendSize = new Semaphore(1024 * 1024, true); + log.info("semaphoreAsyncSendSize can not be smaller than 1M."); + } +- } + ++ ServiceDetector serviceDetector = new ServiceDetector() { ++ @Override ++ public boolean detect(String endpoint, long timeoutMillis) { ++ Optional candidateTopic = pickTopic(); ++ if (!candidateTopic.isPresent()) { ++ return false; ++ } ++ try { ++ MessageQueue mq = new MessageQueue(candidateTopic.get(), null, 0); ++ mQClientFactory.getMQClientAPIImpl() ++ .getMaxOffset(endpoint, mq, timeoutMillis); ++ return true; ++ } catch (Exception e) { ++ return false; ++ } ++ } ++ }; ++ ++ this.mqFaultStrategy = new MQFaultStrategy(defaultMQProducer.cloneClientConfig(), new Resolver() { ++ @Override ++ public String resolve(String name) { ++ return DefaultMQProducerImpl.this.mQClientFactory.findBrokerAddressInPublish(name); ++ } ++ }, serviceDetector); ++ } ++ private Optional pickTopic() { ++ if (topicPublishInfoTable.isEmpty()) { ++ return Optional.absent(); ++ } ++ return Optional.of(topicPublishInfoTable.keySet().iterator().next()); ++ } + public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { + this.checkForbiddenHookList.add(checkForbiddenHook); + log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(), +@@ -229,6 +263,10 @@ public class DefaultMQProducerImpl implements MQProducerInner { + mQClientFactory.start(); + } + ++ if (this.mqFaultStrategy.isStartDetectorEnable()) { ++ this.mqFaultStrategy.startDetector(); ++ } ++ + log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), + this.defaultMQProducer.isSendMessageWithVIPChannel()); + this.serviceState = ServiceState.RUNNING; +@@ -273,6 +311,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { + if (shutdownFactory) { + this.mQClientFactory.shutdown(); + } ++ if (this.mqFaultStrategy.isStartDetectorEnable()) { ++ this.mqFaultStrategy.shutdown(); ++ } + RequestFutureHolder.getInstance().shutdown(this); + log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; +@@ -574,7 +615,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + } + + public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg, +- final long timeout) throws MQClientException, RemotingTooMuchRequestException { ++ final long timeout) throws MQClientException, RemotingTooMuchRequestException { + long beginStartTime = System.currentTimeMillis(); + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); +@@ -584,7 +625,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + MessageQueue mq = null; + try { + List messageQueueList = +- mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); ++ mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); + Message userMessage = MessageAccessor.cloneMessage(msg); + String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace()); + userMessage.setTopic(userTopic); +@@ -609,12 +650,13 @@ public class DefaultMQProducerImpl implements MQProducerInner { + throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); + } + +- public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { +- return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); ++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { ++ return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName, resetIndex); + } + +- public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { +- this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); ++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, ++ boolean reachable) { ++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); + } + + private void validateNameServerSetting() throws MQClientException { +@@ -647,9 +689,13 @@ public class DefaultMQProducerImpl implements MQProducerInner { + int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; + int times = 0; + String[] brokersSent = new String[timesTotal]; ++ boolean resetIndex = false; + for (; times < timesTotal; times++) { + String lastBrokerName = null == mq ? null : mq.getBrokerName(); +- MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); ++ if (times > 0) { ++ resetIndex = true; ++ } ++ MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName, resetIndex); + if (mqSelected != null) { + mq = mqSelected; + brokersSent[times] = mq.getBrokerName(); +@@ -667,7 +713,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + + sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); + endTimestamp = System.currentTimeMillis(); +- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); ++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); + switch (communicationMode) { + case ASYNC: + return null; +@@ -684,9 +730,22 @@ public class DefaultMQProducerImpl implements MQProducerInner { + default: + break; + } +- } catch (RemotingException | MQClientException e) { ++ } catch (MQClientException e) { + endTimestamp = System.currentTimeMillis(); +- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); ++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); ++ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); ++ log.warn(msg.toString()); ++ exception = e; ++ continue; ++ } catch (RemotingException e) { ++ endTimestamp = System.currentTimeMillis(); ++ if (this.mqFaultStrategy.isStartDetectorEnable()) { ++ // Set this broker unreachable when detecting schedule task is running for RemotingException. ++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false); ++ } else { ++ // Otherwise, isolate this broker. ++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, true); ++ } + log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); + if (log.isDebugEnabled()) { + log.debug(msg.toString()); +@@ -695,7 +754,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + continue; + } catch (MQBrokerException e) { + endTimestamp = System.currentTimeMillis(); +- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); ++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false); + log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); + if (log.isDebugEnabled()) { + log.debug(msg.toString()); +@@ -712,7 +771,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + } + } catch (InterruptedException e) { + endTimestamp = System.currentTimeMillis(); +- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); ++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); + log.warn("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); + if (log.isDebugEnabled()) { + log.debug(msg.toString()); +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java +index 275ada7ac..37b1f3252 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java +@@ -18,6 +18,8 @@ package org.apache.rocketmq.client.impl.producer; + + import java.util.ArrayList; + import java.util.List; ++ ++import com.google.common.base.Preconditions; + import org.apache.rocketmq.client.common.ThreadLocalIndex; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.remoting.protocol.route.QueueData; +@@ -30,6 +32,10 @@ public class TopicPublishInfo { + private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex(); + private TopicRouteData topicRouteData; + ++ public interface QueueFilter { ++ boolean filter(MessageQueue mq); ++ } ++ + public boolean isOrderTopic() { + return orderTopic; + } +@@ -66,6 +72,40 @@ public class TopicPublishInfo { + this.haveTopicRouterInfo = haveTopicRouterInfo; + } + ++ public MessageQueue selectOneMessageQueue(QueueFilter ...filter) { ++ return selectOneMessageQueue(this.messageQueueList, this.sendWhichQueue, filter); ++ } ++ ++ private MessageQueue selectOneMessageQueue(List messageQueueList, ThreadLocalIndex sendQueue, QueueFilter ...filter) { ++ if (messageQueueList == null || messageQueueList.isEmpty()) { ++ return null; ++ } ++ ++ if (filter != null && filter.length != 0) { ++ for (int i = 0; i < messageQueueList.size(); i++) { ++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); ++ MessageQueue mq = messageQueueList.get(index); ++ boolean filterResult = true; ++ for (QueueFilter f: filter) { ++ Preconditions.checkNotNull(f); ++ filterResult &= f.filter(mq); ++ } ++ if (filterResult) { ++ return mq; ++ } ++ } ++ ++ return null; ++ } ++ ++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); ++ return messageQueueList.get(index); ++ } ++ ++ public void resetIndex() { ++ this.sendWhichQueue.reset(); ++ } ++ + public MessageQueue selectOneMessageQueue(final String lastBrokerName) { + if (lastBrokerName == null) { + return selectOneMessageQueue(); +diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java +index 09a8aa461..72d2f3450 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java +@@ -18,11 +18,75 @@ + package org.apache.rocketmq.client.latency; + + public interface LatencyFaultTolerance { +- void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration); ++ /** ++ * Update brokers' states, to decide if they are good or not. ++ * ++ * @param name Broker's name. ++ * @param currentLatency Current message sending process's latency. ++ * @param notAvailableDuration Corresponding not available time, ms. The broker will be not available until it ++ * spends such time. ++ * @param reachable To decide if this broker is reachable or not. ++ */ ++ void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration, ++ final boolean reachable); + ++ /** ++ * To check if this broker is available. ++ * ++ * @param name Broker's name. ++ * @return boolean variable, if this is true, then the broker is available. ++ */ + boolean isAvailable(final T name); + ++ /** ++ * To check if this broker is reachable. ++ * ++ * @param name Broker's name. ++ * @return boolean variable, if this is true, then the broker is reachable. ++ */ ++ boolean isReachable(final T name); ++ ++ /** ++ * Remove the broker in this fault item table. ++ * ++ * @param name broker's name. ++ */ + void remove(final T name); + ++ /** ++ * The worst situation, no broker can be available. Then choose random one. ++ * ++ * @return A random mq will be returned. ++ */ + T pickOneAtLeast(); ++ ++ /** ++ * Start a new thread, to detect the broker's reachable tag. ++ */ ++ void startDetector(); ++ ++ /** ++ * Shutdown threads that started by LatencyFaultTolerance. ++ */ ++ void shutdown(); ++ ++ /** ++ * A function reserved, just detect by once, won't create a new thread. ++ */ ++ void detectByOneRound(); ++ ++ /** ++ * Use it to set the detect timeout bound. ++ * ++ * @param detectTimeout timeout bound ++ */ ++ void setDetectTimeout(final int detectTimeout); ++ ++ /** ++ * Use it to set the detector's detector interval for each broker (each broker will be detected once during this ++ * time) ++ * ++ * @param detectInterval each broker's detecting interval ++ */ ++ void setDetectInterval(final int detectInterval); + } +diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +index 93795d957..8af629574 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +@@ -21,30 +21,97 @@ import java.util.Collections; + import java.util.Enumeration; + import java.util.LinkedList; + import java.util.List; ++import java.util.Map; + import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.Executors; ++import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.ThreadFactory; ++import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.client.common.ThreadLocalIndex; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + + public class LatencyFaultToleranceImpl implements LatencyFaultTolerance { +- private final ConcurrentHashMap faultItemTable = new ConcurrentHashMap<>(16); ++ private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class); ++ private final ConcurrentHashMap faultItemTable = new ConcurrentHashMap(16); ++ private int detectTimeout = 200; ++ private int detectInterval = 2000; ++ private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex(); ++ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { ++ @Override ++ public Thread newThread(Runnable r) { ++ return new Thread(r, "LatencyFaultToleranceScheduledThread"); ++ } ++ }); + +- private final ThreadLocalIndex randomItem = new ThreadLocalIndex(); ++ private final Resolver resolver; ++ ++ private final ServiceDetector serviceDetector; ++ ++ public LatencyFaultToleranceImpl(Resolver resolver, ServiceDetector serviceDetector) { ++ this.resolver = resolver; ++ this.serviceDetector = serviceDetector; ++ } ++ ++ public void detectByOneRound() { ++ for (Map.Entry item : this.faultItemTable.entrySet()) { ++ FaultItem brokerItem = item.getValue(); ++ if (System.currentTimeMillis() - brokerItem.checkStamp >= 0) { ++ brokerItem.checkStamp = System.currentTimeMillis() + this.detectInterval; ++ String brokerAddr = resolver.resolve(brokerItem.getName()); ++ if (brokerAddr == null) { ++ faultItemTable.remove(item.getKey()); ++ continue; ++ } ++ if (null == serviceDetector) { ++ continue; ++ } ++ boolean serviceOK = serviceDetector.detect(brokerAddr, detectTimeout); ++ if (serviceOK && !brokerItem.reachableFlag) { ++ log.info(brokerItem.name + " is reachable now, then it can be used."); ++ brokerItem.reachableFlag = true; ++ } ++ } ++ } ++ } ++ ++ public void startDetector() { ++ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { ++ @Override ++ public void run() { ++ try { ++ detectByOneRound(); ++ } catch (Exception e) { ++ log.warn("Unexpected exception raised while detecting service reachability", e); ++ } ++ } ++ }, 3, 3, TimeUnit.SECONDS); ++ } ++ ++ public void shutdown() { ++ this.scheduledExecutorService.shutdown(); ++ } + + @Override +- public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration) { ++ public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration, ++ final boolean reachable) { + FaultItem old = this.faultItemTable.get(name); + if (null == old) { + final FaultItem faultItem = new FaultItem(name); + faultItem.setCurrentLatency(currentLatency); +- faultItem.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); +- ++ faultItem.updateNotAvailableDuration(notAvailableDuration); ++ faultItem.setReachable(reachable); + old = this.faultItemTable.putIfAbsent(name, faultItem); +- if (old != null) { +- old.setCurrentLatency(currentLatency); +- old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); +- } +- } else { ++ } ++ ++ if (null != old) { + old.setCurrentLatency(currentLatency); +- old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); ++ old.updateNotAvailableDuration(notAvailableDuration); ++ old.setReachable(reachable); ++ } ++ ++ if (!reachable) { ++ log.info(name + " is unreachable, it will not be used until it's reachable"); + } + } + +@@ -57,6 +124,14 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance + return true; + } + ++ public boolean isReachable(final String name) { ++ final FaultItem faultItem = this.faultItemTable.get(name); ++ if (faultItem != null) { ++ return faultItem.isReachable(); ++ } ++ return true; ++ } ++ + @Override + public void remove(final String name) { + this.faultItemTable.remove(name); +@@ -65,68 +140,98 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance + @Override + public String pickOneAtLeast() { + final Enumeration elements = this.faultItemTable.elements(); +- List tmpList = new LinkedList<>(); ++ List tmpList = new LinkedList(); + while (elements.hasMoreElements()) { + final FaultItem faultItem = elements.nextElement(); + tmpList.add(faultItem); + } ++ + if (!tmpList.isEmpty()) { +- Collections.sort(tmpList); +- final int half = tmpList.size() / 2; +- if (half <= 0) { +- return tmpList.get(0).getName(); +- } else { +- final int i = this.randomItem.incrementAndGet() % half; +- return tmpList.get(i).getName(); ++ Collections.shuffle(tmpList); ++ for (FaultItem faultItem : tmpList) { ++ if (faultItem.reachableFlag) { ++ return faultItem.name; ++ } + } + } ++ + return null; + } + + @Override + public String toString() { + return "LatencyFaultToleranceImpl{" + +- "faultItemTable=" + faultItemTable + +- ", whichItemWorst=" + randomItem + +- '}'; ++ "faultItemTable=" + faultItemTable + ++ ", whichItemWorst=" + whichItemWorst + ++ '}'; ++ } ++ ++ public void setDetectTimeout(final int detectTimeout) { ++ this.detectTimeout = detectTimeout; + } + +- class FaultItem implements Comparable { ++ public void setDetectInterval(final int detectInterval) { ++ this.detectInterval = detectInterval; ++ } ++ ++ public class FaultItem implements Comparable { + private final String name; + private volatile long currentLatency; + private volatile long startTimestamp; ++ private volatile long checkStamp; ++ private volatile boolean reachableFlag; + + public FaultItem(final String name) { + this.name = name; + } + ++ public void updateNotAvailableDuration(long notAvailableDuration) { ++ if (notAvailableDuration > 0 && System.currentTimeMillis() + notAvailableDuration > this.startTimestamp) { ++ this.startTimestamp = System.currentTimeMillis() + notAvailableDuration; ++ log.info(name + " will be isolated for " + notAvailableDuration + " ms."); ++ } ++ } ++ + @Override + public int compareTo(final FaultItem other) { + if (this.isAvailable() != other.isAvailable()) { +- if (this.isAvailable()) ++ if (this.isAvailable()) { + return -1; ++ } + +- if (other.isAvailable()) ++ if (other.isAvailable()) { + return 1; ++ } + } + +- if (this.currentLatency < other.currentLatency) ++ if (this.currentLatency < other.currentLatency) { + return -1; +- else if (this.currentLatency > other.currentLatency) { ++ } else if (this.currentLatency > other.currentLatency) { + return 1; + } + +- if (this.startTimestamp < other.startTimestamp) ++ if (this.startTimestamp < other.startTimestamp) { + return -1; +- else if (this.startTimestamp > other.startTimestamp) { ++ } else if (this.startTimestamp > other.startTimestamp) { + return 1; + } +- + return 0; + } + ++ public void setReachable(boolean reachableFlag) { ++ this.reachableFlag = reachableFlag; ++ } ++ ++ public void setCheckStamp(long checkStamp) { ++ this.checkStamp = checkStamp; ++ } ++ + public boolean isAvailable() { +- return (System.currentTimeMillis() - startTimestamp) >= 0; ++ return reachableFlag && System.currentTimeMillis() >= startTimestamp; ++ } ++ ++ public boolean isReachable() { ++ return reachableFlag; + } + + @Override +@@ -139,28 +244,32 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance + + @Override + public boolean equals(final Object o) { +- if (this == o) ++ if (this == o) { + return true; +- if (!(o instanceof FaultItem)) ++ } ++ if (!(o instanceof FaultItem)) { + return false; ++ } + + final FaultItem faultItem = (FaultItem) o; + +- if (getCurrentLatency() != faultItem.getCurrentLatency()) ++ if (getCurrentLatency() != faultItem.getCurrentLatency()) { + return false; +- if (getStartTimestamp() != faultItem.getStartTimestamp()) ++ } ++ if (getStartTimestamp() != faultItem.getStartTimestamp()) { + return false; ++ } + return getName() != null ? getName().equals(faultItem.getName()) : faultItem.getName() == null; +- + } + + @Override + public String toString() { + return "FaultItem{" + +- "name='" + name + '\'' + +- ", currentLatency=" + currentLatency + +- ", startTimestamp=" + startTimestamp + +- '}'; ++ "name='" + name + '\'' + ++ ", currentLatency=" + currentLatency + ++ ", startTimestamp=" + startTimestamp + ++ ", reachableFlag=" + reachableFlag + ++ '}'; + } + + public String getName() { +diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +index 1e1953fad..c01490784 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +@@ -17,25 +17,86 @@ + + package org.apache.rocketmq.client.latency; + +-import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.client.ClientConfig; + import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; ++import org.apache.rocketmq.client.impl.producer.TopicPublishInfo.QueueFilter; + import org.apache.rocketmq.common.message.MessageQueue; +-import org.apache.rocketmq.logging.org.slf4j.Logger; +-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + + public class MQFaultStrategy { +- private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class); +- private final LatencyFaultTolerance latencyFaultTolerance = new LatencyFaultToleranceImpl(); ++ private LatencyFaultTolerance latencyFaultTolerance; ++ private boolean sendLatencyFaultEnable; ++ private boolean startDetectorEnable; ++ private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L}; ++ private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L}; + +- private boolean sendLatencyFaultEnable = false; ++ public static class BrokerFilter implements QueueFilter { ++ private String lastBrokerName; ++ ++ public void setLastBrokerName(String lastBrokerName) { ++ this.lastBrokerName = lastBrokerName; ++ } ++ ++ @Override public boolean filter(MessageQueue mq) { ++ if (lastBrokerName != null) { ++ return !mq.getBrokerName().equals(lastBrokerName); ++ } ++ return true; ++ } ++ } ++ ++ private ThreadLocal threadBrokerFilter = new ThreadLocal() { ++ @Override protected BrokerFilter initialValue() { ++ return new BrokerFilter(); ++ } ++ }; ++ ++ private QueueFilter reachableFilter = new QueueFilter() { ++ @Override public boolean filter(MessageQueue mq) { ++ return latencyFaultTolerance.isReachable(mq.getBrokerName()); ++ } ++ }; ++ ++ private QueueFilter availableFilter = new QueueFilter() { ++ @Override public boolean filter(MessageQueue mq) { ++ return latencyFaultTolerance.isAvailable(mq.getBrokerName()); ++ } ++ }; ++ ++ ++ public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) { ++ this.setStartDetectorEnable(cc.isStartDetectorEnable()); ++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); ++ this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector); ++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); ++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); ++ } ++ ++ // For unit test. ++ public MQFaultStrategy(ClientConfig cc, LatencyFaultTolerance tolerance) { ++ this.setStartDetectorEnable(cc.isStartDetectorEnable()); ++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); ++ this.latencyFaultTolerance = tolerance; ++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); ++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); ++ } + +- private long[] latencyMax = {50L, 100L, 550L, 1000L, 2000L, 3000L, 15000L}; +- private long[] notAvailableDuration = {0L, 0L, 30000L, 60000L, 120000L, 180000L, 600000L}; + + public long[] getNotAvailableDuration() { + return notAvailableDuration; + } + ++ public QueueFilter getAvailableFilter() { ++ return availableFilter; ++ } ++ ++ public QueueFilter getReachableFilter() { ++ return reachableFilter; ++ } ++ ++ public ThreadLocal getThreadBrokerFilter() { ++ return threadBrokerFilter; ++ } ++ + public void setNotAvailableDuration(final long[] notAvailableDuration) { + this.notAvailableDuration = notAvailableDuration; + } +@@ -56,51 +117,69 @@ public class MQFaultStrategy { + this.sendLatencyFaultEnable = sendLatencyFaultEnable; + } + +- public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { ++ public boolean isStartDetectorEnable() { ++ return startDetectorEnable; ++ } ++ ++ public void setStartDetectorEnable(boolean startDetectorEnable) { ++ this.startDetectorEnable = startDetectorEnable; ++ } ++ ++ public void startDetector() { ++ // user should start the detector ++ // and the thread should not be in running state. ++ if (this.sendLatencyFaultEnable && this.startDetectorEnable) { ++ // start the detector. ++ this.latencyFaultTolerance.startDetector(); ++ } ++ } ++ ++ public void shutdown() { ++ if (this.sendLatencyFaultEnable && this.startDetectorEnable) { ++ this.latencyFaultTolerance.shutdown(); ++ } ++ } ++ ++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { ++ BrokerFilter brokerFilter = threadBrokerFilter.get(); ++ brokerFilter.setLastBrokerName(lastBrokerName); + if (this.sendLatencyFaultEnable) { +- try { +- int index = tpInfo.getSendWhichQueue().incrementAndGet(); +- for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) { +- int pos = index++ % tpInfo.getMessageQueueList().size(); +- MessageQueue mq = tpInfo.getMessageQueueList().get(pos); +- if (!StringUtils.equals(lastBrokerName, mq.getBrokerName()) && latencyFaultTolerance.isAvailable(mq.getBrokerName())) { +- return mq; +- } +- } +- +- final String notBestBroker = latencyFaultTolerance.pickOneAtLeast(); +- int writeQueueNums = tpInfo.getWriteQueueNumsByBroker(notBestBroker); +- if (writeQueueNums > 0) { +- final MessageQueue mq = tpInfo.selectOneMessageQueue(); +- if (notBestBroker != null) { +- mq.setBrokerName(notBestBroker); +- mq.setQueueId(tpInfo.getSendWhichQueue().incrementAndGet() % writeQueueNums); +- } +- return mq; +- } else { +- latencyFaultTolerance.remove(notBestBroker); +- } +- } catch (Exception e) { +- log.error("Error occurred when selecting message queue", e); ++ if (resetIndex) { ++ tpInfo.resetIndex(); ++ } ++ MessageQueue mq = tpInfo.selectOneMessageQueue(availableFilter, brokerFilter); ++ if (mq != null) { ++ return mq; ++ } ++ ++ mq = tpInfo.selectOneMessageQueue(reachableFilter, brokerFilter); ++ if (mq != null) { ++ return mq; + } + + return tpInfo.selectOneMessageQueue(); + } + +- return tpInfo.selectOneMessageQueue(lastBrokerName); ++ MessageQueue mq = tpInfo.selectOneMessageQueue(brokerFilter); ++ if (mq != null) { ++ return mq; ++ } ++ return tpInfo.selectOneMessageQueue(); + } + +- public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { ++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, ++ final boolean reachable) { + if (this.sendLatencyFaultEnable) { +- long duration = computeNotAvailableDuration(isolation ? 30000 : currentLatency); +- this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration); ++ long duration = computeNotAvailableDuration(isolation ? 10000 : currentLatency); ++ this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration, reachable); + } + } + + private long computeNotAvailableDuration(final long currentLatency) { + for (int i = latencyMax.length - 1; i >= 0; i--) { +- if (currentLatency >= latencyMax[i]) ++ if (currentLatency >= latencyMax[i]) { + return this.notAvailableDuration[i]; ++ } + } + + return 0; +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java +similarity index 65% +rename from remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java +rename to client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java +index 6aa547047..1c29ba334 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java ++++ b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java +@@ -14,20 +14,9 @@ + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-package org.apache.rocketmq.remoting.protocol.body; ++package org.apache.rocketmq.client.latency; + +-import java.util.ArrayList; +-import java.util.List; +-import org.apache.rocketmq.remoting.protocol.RemotingSerializable; ++public interface Resolver { + +-public class GetRemoteClientConfigBody extends RemotingSerializable { +- private List keys = new ArrayList<>(); +- +- public List getKeys() { +- return keys; +- } +- +- public void setKeys(List keys) { +- this.keys = keys; +- } ++ String resolve(String name); + } +diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java +new file mode 100644 +index 000000000..c6ffbad1c +--- /dev/null ++++ b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java +@@ -0,0 +1,30 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.client.latency; ++ ++/** ++ * Detect whether the remote service state is normal. ++ */ ++public interface ServiceDetector { ++ ++ /** ++ * Check if the remote service is normal. ++ * @param endpoint Service endpoint to check against ++ * @return true if the service is back to normal; false otherwise. ++ */ ++ boolean detect(String endpoint, long timeoutMillis); ++} +diff --git a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java +index 86690e40b..42ccdae5a 100644 +--- a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java ++++ b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java +@@ -16,11 +16,14 @@ + */ + package org.apache.rocketmq.client.latency; + +-import java.util.concurrent.TimeUnit; ++import org.awaitility.core.ThrowingRunnable; + import org.junit.Before; + import org.junit.Test; + ++import java.util.concurrent.TimeUnit; ++ + import static org.assertj.core.api.Assertions.assertThat; ++import static org.awaitility.Awaitility.await; + + public class LatencyFaultToleranceImplTest { + private LatencyFaultTolerance latencyFaultTolerance; +@@ -29,28 +32,31 @@ public class LatencyFaultToleranceImplTest { + + @Before + public void init() { +- latencyFaultTolerance = new LatencyFaultToleranceImpl(); ++ latencyFaultTolerance = new LatencyFaultToleranceImpl(null, null); + } + + @Test + public void testUpdateFaultItem() throws Exception { +- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000); ++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true); + assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); + assertThat(latencyFaultTolerance.isAvailable(anotherBrokerName)).isTrue(); + } + + @Test + public void testIsAvailable() throws Exception { +- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50); ++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50, true); + assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); + +- TimeUnit.MILLISECONDS.sleep(70); +- assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); ++ await().atMost(500, TimeUnit.MILLISECONDS).untilAsserted(new ThrowingRunnable() { ++ @Override public void run() throws Throwable { ++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); ++ } ++ }); + } + + @Test + public void testRemove() throws Exception { +- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000); ++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true); + assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); + latencyFaultTolerance.remove(brokerName); + assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); +@@ -58,10 +64,20 @@ public class LatencyFaultToleranceImplTest { + + @Test + public void testPickOneAtLeast() throws Exception { +- latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000); ++ latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true); + assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); + +- latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000); +- assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); ++ // Bad case, since pickOneAtLeast's behavior becomes random ++ // latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, "127.0.0.1:12011", true); ++ // assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); ++ } ++ ++ @Test ++ public void testIsReachable() throws Exception { ++ latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true); ++ assertThat(latencyFaultTolerance.isReachable(brokerName)).isEqualTo(true); ++ ++ latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, false); ++ assertThat(latencyFaultTolerance.isReachable(anotherBrokerName)).isEqualTo(false); + } + } +\ No newline at end of file +diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +index fada0efd7..485b95c42 100644 +--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java ++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +@@ -41,7 +41,6 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; + import org.apache.rocketmq.remoting.protocol.ResponseCode; + import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; + import org.apache.rocketmq.remoting.protocol.body.GetBrokerMemberGroupResponseBody; +-import org.apache.rocketmq.remoting.protocol.body.GetRemoteClientConfigBody; + import org.apache.rocketmq.remoting.protocol.body.RegisterBrokerBody; + import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; + import org.apache.rocketmq.remoting.protocol.body.TopicList; +@@ -132,8 +131,6 @@ public class DefaultRequestProcessor implements NettyRequestProcessor { + return this.updateConfig(ctx, request); + case RequestCode.GET_NAMESRV_CONFIG: + return this.getConfig(ctx, request); +- case RequestCode.GET_CLIENT_CONFIG: +- return this.getClientConfigs(ctx, request); + default: + String error = " request type " + request.getCode() + " not supported"; + return RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error); +@@ -661,25 +658,4 @@ public class DefaultRequestProcessor implements NettyRequestProcessor { + return response; + } + +- private RemotingCommand getClientConfigs(ChannelHandlerContext ctx, RemotingCommand request) { +- final RemotingCommand response = RemotingCommand.createResponseCommand(null); +- final GetRemoteClientConfigBody body = GetRemoteClientConfigBody.decode(request.getBody(), GetRemoteClientConfigBody.class); +- +- String content = this.namesrvController.getConfiguration().getClientConfigsFormatString(body.getKeys()); +- if (StringUtils.isNotBlank(content)) { +- try { +- response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); +- } catch (UnsupportedEncodingException e) { +- log.error("getConfig error, ", e); +- response.setCode(ResponseCode.SYSTEM_ERROR); +- response.setRemark("UnsupportedEncodingException " + e); +- return response; +- } +- } +- +- response.setCode(ResponseCode.SUCCESS); +- response.setRemark(null); +- return response; +- } +- + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +index 2994893d7..b2478fec3 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +@@ -232,6 +232,12 @@ public class ProxyConfig implements ConfigFile { + private String remotingAccessAddr = ""; + private int remotingListenPort = 8080; + ++ // related to proxy's send strategy in cluster mode. ++ private boolean sendLatencyEnable = false; ++ private boolean startDetectorEnable = false; ++ private int detectTimeout = 200; ++ private int detectInterval = 2 * 1000; ++ + private int remotingHeartbeatThreadPoolNums = 2 * PROCESSOR_NUMBER; + private int remotingTopicRouteThreadPoolNums = 2 * PROCESSOR_NUMBER; + private int remotingSendMessageThreadPoolNums = 4 * PROCESSOR_NUMBER; +@@ -1409,6 +1415,46 @@ public class ProxyConfig implements ConfigFile { + this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; + } + ++ public boolean isSendLatencyEnable() { ++ return sendLatencyEnable; ++ } ++ ++ public boolean isStartDetectorEnable() { ++ return startDetectorEnable; ++ } ++ ++ public void setStartDetectorEnable(boolean startDetectorEnable) { ++ this.startDetectorEnable = startDetectorEnable; ++ } ++ ++ public void setSendLatencyEnable(boolean sendLatencyEnable) { ++ this.sendLatencyEnable = sendLatencyEnable; ++ } ++ ++ public boolean getStartDetectorEnable() { ++ return this.startDetectorEnable; ++ } ++ ++ public boolean getSendLatencyEnable() { ++ return this.sendLatencyEnable; ++ } ++ ++ public int getDetectTimeout() { ++ return detectTimeout; ++ } ++ ++ public void setDetectTimeout(int detectTimeout) { ++ this.detectTimeout = detectTimeout; ++ } ++ ++ public int getDetectInterval() { ++ return detectInterval; ++ } ++ ++ public void setDetectInterval(int detectInterval) { ++ this.detectInterval = detectInterval; ++ } ++ + public boolean isEnableBatchAck() { + return enableBatchAck; + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java +index 6146c80cd..f670df205 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java +@@ -382,7 +382,7 @@ public class SendMessageActivity extends AbstractMessingActivity { + int bucket = Hashing.consistentHash(shardingKey.hashCode(), writeQueues.size()); + targetMessageQueue = writeQueues.get(bucket); + } else { +- targetMessageQueue = messageQueueView.getWriteSelector().selectOne(false); ++ targetMessageQueue = messageQueueView.getWriteSelector().selectOneByPipeline(false); + } + return targetMessageQueue; + } catch (Exception e) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java +index 0d0c62168..a80f6df0b 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java +@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.processor; + import java.util.List; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ExecutorService; ++ + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.client.producer.SendResult; + import org.apache.rocketmq.client.producer.SendStatus; +@@ -66,6 +67,8 @@ public class ProducerProcessor extends AbstractProcessor { + public CompletableFuture> sendMessage(ProxyContext ctx, QueueSelector queueSelector, + String producerGroup, int sysFlag, List messageList, long timeoutMillis) { + CompletableFuture> future = new CompletableFuture<>(); ++ long beginTimestampFirst = System.currentTimeMillis(); ++ AddressableMessageQueue messageQueue = null; + try { + Message message = messageList.get(0); + String topic = message.getTopic(); +@@ -79,7 +82,7 @@ public class ProducerProcessor extends AbstractProcessor { + } + } + } +- AddressableMessageQueue messageQueue = queueSelector.select(ctx, ++ messageQueue = queueSelector.select(ctx, + this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(ctx, topic)); + if (messageQueue == null) { + throw new ProxyException(ProxyExceptionCode.FORBIDDEN, "no writable queue"); +@@ -90,6 +93,7 @@ public class ProducerProcessor extends AbstractProcessor { + } + SendMessageRequestHeader requestHeader = buildSendMessageRequestHeader(messageList, producerGroup, sysFlag, messageQueue.getQueueId()); + ++ AddressableMessageQueue finalMessageQueue = messageQueue; + future = this.serviceManager.getMessageService().sendMessage( + ctx, + messageQueue, +@@ -102,11 +106,19 @@ public class ProducerProcessor extends AbstractProcessor { + if (SendStatus.SEND_OK.equals(sendResult.getSendStatus()) && + tranType == MessageSysFlag.TRANSACTION_PREPARED_TYPE && + StringUtils.isNotBlank(sendResult.getTransactionId())) { +- fillTransactionData(ctx, producerGroup, messageQueue, sendResult, messageList); ++ fillTransactionData(ctx, producerGroup, finalMessageQueue, sendResult, messageList); + } + } + return sendResultList; +- }, this.executor); ++ }, this.executor) ++ .whenComplete((result, exception) -> { ++ long endTimestamp = System.currentTimeMillis(); ++ if (exception != null) { ++ this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(), endTimestamp - beginTimestampFirst, true, false); ++ } else { ++ this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(),endTimestamp - beginTimestampFirst, false, true); ++ } ++ }); + } catch (Throwable t) { + future.completeExceptionally(t); + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java +index d67b68f38..aced15cee 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java +@@ -54,7 +54,7 @@ public class LocalTopicRouteService extends TopicRouteService { + @Override + public MessageQueueView getCurrentMessageQueueView(ProxyContext ctx, String topic) throws Exception { + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic); +- return new MessageQueueView(topic, toTopicRouteData(topicConfig)); ++ return new MessageQueueView(topic, toTopicRouteData(topicConfig), null); + } + + @Override +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java +index 85cd18d45..f25fb907e 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java +@@ -17,6 +17,7 @@ + package org.apache.rocketmq.proxy.service.route; + + import com.google.common.base.MoreObjects; ++import com.google.common.base.Preconditions; + import com.google.common.math.IntMath; + import java.util.ArrayList; + import java.util.Collections; +@@ -30,6 +31,8 @@ import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.atomic.AtomicInteger; + import java.util.stream.Collectors; + import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; ++import org.apache.rocketmq.client.latency.MQFaultStrategy; + import org.apache.rocketmq.common.constant.PermName; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.remoting.protocol.route.QueueData; +@@ -44,8 +47,9 @@ public class MessageQueueSelector { + private final Map brokerNameQueueMap = new ConcurrentHashMap<>(); + private final AtomicInteger queueIndex; + private final AtomicInteger brokerIndex; ++ private MQFaultStrategy mqFaultStrategy; + +- public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, boolean read) { ++ public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, MQFaultStrategy mqFaultStrategy, boolean read) { + if (read) { + this.queues.addAll(buildRead(topicRouteWrapper)); + } else { +@@ -55,6 +59,7 @@ public class MessageQueueSelector { + Random random = new Random(); + this.queueIndex = new AtomicInteger(random.nextInt()); + this.brokerIndex = new AtomicInteger(random.nextInt()); ++ this.mqFaultStrategy = mqFaultStrategy; + } + + private static List buildRead(TopicRouteWrapper topicRoute) { +@@ -154,6 +159,86 @@ public class MessageQueueSelector { + return selectOneByIndex(nextIndex, onlyBroker); + } + ++ public AddressableMessageQueue selectOneByPipeline(boolean onlyBroker) { ++ if (mqFaultStrategy != null && mqFaultStrategy.isSendLatencyFaultEnable()) { ++ List messageQueueList = null; ++ MessageQueue messageQueue = null; ++ if (onlyBroker) { ++ messageQueueList = transferAddressableQueues(brokerActingQueues); ++ } else { ++ messageQueueList = transferAddressableQueues(queues); ++ } ++ AddressableMessageQueue addressableMessageQueue = null; ++ ++ // use both available filter. ++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, ++ mqFaultStrategy.getAvailableFilter(), mqFaultStrategy.getReachableFilter()); ++ addressableMessageQueue = transferQueue2Addressable(messageQueue); ++ if (addressableMessageQueue != null) { ++ return addressableMessageQueue; ++ } ++ ++ // use available filter. ++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, ++ mqFaultStrategy.getAvailableFilter()); ++ addressableMessageQueue = transferQueue2Addressable(messageQueue); ++ if (addressableMessageQueue != null) { ++ return addressableMessageQueue; ++ } ++ ++ // no available filter, then use reachable filter. ++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, ++ mqFaultStrategy.getReachableFilter()); ++ addressableMessageQueue = transferQueue2Addressable(messageQueue); ++ if (addressableMessageQueue != null) { ++ return addressableMessageQueue; ++ } ++ } ++ ++ // SendLatency is not enabled, or no queue is selected, then select by index. ++ return selectOne(onlyBroker); ++ } ++ ++ private MessageQueue selectOneMessageQueue(List messageQueueList, AtomicInteger sendQueue, TopicPublishInfo.QueueFilter...filter) { ++ if (messageQueueList == null || messageQueueList.isEmpty()) { ++ return null; ++ } ++ if (filter != null && filter.length != 0) { ++ for (int i = 0; i < messageQueueList.size(); i++) { ++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); ++ MessageQueue mq = messageQueueList.get(index); ++ boolean filterResult = true; ++ for (TopicPublishInfo.QueueFilter f: filter) { ++ Preconditions.checkNotNull(f); ++ filterResult &= f.filter(mq); ++ } ++ if (filterResult) { ++ return mq; ++ } ++ } ++ } ++ return null; ++ } ++ ++ public List transferAddressableQueues(List addressableMessageQueueList) { ++ if (addressableMessageQueueList == null) { ++ return null; ++ } ++ ++ return addressableMessageQueueList.stream() ++ .map(AddressableMessageQueue::getMessageQueue) ++ .collect(Collectors.toList()); ++ } ++ ++ private AddressableMessageQueue transferQueue2Addressable(MessageQueue messageQueue) { ++ for (AddressableMessageQueue amq: queues) { ++ if (amq.getMessageQueue().equals(messageQueue)) { ++ return amq; ++ } ++ } ++ return null; ++ } ++ + public AddressableMessageQueue selectNextOne(AddressableMessageQueue last) { + boolean onlyBroker = last.getQueueId() < 0; + AddressableMessageQueue newOne = last; +@@ -190,6 +275,14 @@ public class MessageQueueSelector { + return brokerActingQueues; + } + ++ public MQFaultStrategy getMQFaultStrategy() { ++ return mqFaultStrategy; ++ } ++ ++ public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { ++ this.mqFaultStrategy = mqFaultStrategy; ++ } ++ + @Override + public boolean equals(Object o) { + if (this == o) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java +index fe5387cfd..8b3c2f7c8 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java +@@ -17,20 +17,22 @@ + package org.apache.rocketmq.proxy.service.route; + + import com.google.common.base.MoreObjects; ++import org.apache.rocketmq.client.latency.MQFaultStrategy; + import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; + + public class MessageQueueView { +- public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData()); ++ public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData(), null); + + private final MessageQueueSelector readSelector; + private final MessageQueueSelector writeSelector; + private final TopicRouteWrapper topicRouteWrapper; ++ private MQFaultStrategy mqFaultStrategy; + +- public MessageQueueView(String topic, TopicRouteData topicRouteData) { ++ public MessageQueueView(String topic, TopicRouteData topicRouteData, MQFaultStrategy mqFaultStrategy) { + this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic); + +- this.readSelector = new MessageQueueSelector(topicRouteWrapper, true); +- this.writeSelector = new MessageQueueSelector(topicRouteWrapper, false); ++ this.readSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, true); ++ this.writeSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, false); + } + + public TopicRouteData getTopicRouteData() { +@@ -65,4 +67,12 @@ public class MessageQueueView { + .add("topicRouteWrapper", topicRouteWrapper) + .toString(); + } ++ ++ public MQFaultStrategy getMQFaultStrategy() { ++ return mqFaultStrategy; ++ } ++ ++ public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { ++ this.mqFaultStrategy = mqFaultStrategy; ++ } + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +index 84348adc3..74769a423 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +@@ -25,7 +25,13 @@ import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; ++ ++import com.google.common.base.Optional; ++import org.apache.rocketmq.client.ClientConfig; + import org.apache.rocketmq.client.exception.MQClientException; ++import org.apache.rocketmq.client.latency.MQFaultStrategy; ++import org.apache.rocketmq.client.latency.Resolver; ++import org.apache.rocketmq.client.latency.ServiceDetector; + import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; +@@ -39,6 +45,7 @@ import org.apache.rocketmq.proxy.common.ProxyContext; + import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.config.ProxyConfig; + import org.apache.rocketmq.remoting.protocol.ResponseCode; ++import org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader; + import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; + import org.checkerframework.checker.nullness.qual.NonNull; + import org.checkerframework.checker.nullness.qual.Nullable; +@@ -47,6 +54,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { + private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + private final MQClientAPIFactory mqClientAPIFactory; ++ private MQFaultStrategy mqFaultStrategy; + + protected final LoadingCache topicCache; + protected final ScheduledExecutorService scheduledExecutorService; +@@ -97,15 +105,83 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { + } + } + }); +- ++ ServiceDetector serviceDetector = new ServiceDetector() { ++ @Override ++ public boolean detect(String endpoint, long timeoutMillis) { ++ Optional candidateTopic = pickTopic(); ++ if (!candidateTopic.isPresent()) { ++ return false; ++ } ++ try { ++ GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); ++ requestHeader.setTopic(candidateTopic.get()); ++ requestHeader.setQueueId(0); ++ Long maxOffset = mqClientAPIFactory.getClient().getMaxOffset(endpoint, requestHeader, timeoutMillis).get(); ++ return true; ++ } catch (Exception e) { ++ return false; ++ } ++ } ++ }; ++ mqFaultStrategy = new MQFaultStrategy(extractClientConfigFromProxyConfig(config), new Resolver() { ++ @Override ++ public String resolve(String name) { ++ try { ++ String brokerAddr = getBrokerAddr(null, name); ++ return brokerAddr; ++ } catch (Exception e) { ++ return null; ++ } ++ } ++ }, serviceDetector); + this.init(); + } + ++ // pickup one topic in the topic cache ++ private Optional pickTopic() { ++ if (topicCache.asMap().isEmpty()) { ++ return Optional.absent(); ++ } ++ return Optional.of(topicCache.asMap().keySet().iterator().next()); ++ } ++ + protected void init() { + this.appendShutdown(this.scheduledExecutorService::shutdown); + this.appendStartAndShutdown(this.mqClientAPIFactory); + } + ++ @Override ++ public void shutdown() throws Exception { ++ if (this.mqFaultStrategy.isStartDetectorEnable()) { ++ mqFaultStrategy.shutdown(); ++ } ++ } ++ ++ @Override ++ public void start() throws Exception { ++ if (this.mqFaultStrategy.isStartDetectorEnable()) { ++ this.mqFaultStrategy.startDetector(); ++ } ++ } ++ ++ public ClientConfig extractClientConfigFromProxyConfig(ProxyConfig proxyConfig) { ++ ClientConfig tempClientConfig = new ClientConfig(); ++ tempClientConfig.setSendLatencyEnable(proxyConfig.getSendLatencyEnable()); ++ tempClientConfig.setStartDetectorEnable(proxyConfig.getStartDetectorEnable()); ++ tempClientConfig.setDetectTimeout(proxyConfig.getDetectTimeout()); ++ tempClientConfig.setDetectInterval(proxyConfig.getDetectInterval()); ++ return tempClientConfig; ++ } ++ ++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, ++ boolean reachable) { ++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); ++ } ++ ++ public MQFaultStrategy getMqFaultStrategy() { ++ return this.mqFaultStrategy; ++ } ++ + public MessageQueueView getAllMessageQueueView(ProxyContext ctx, String topicName) throws Exception { + return getCacheMessageQueueWrapper(this.topicCache, topicName); + } +@@ -136,7 +212,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { + + protected MessageQueueView buildMessageQueueView(String topic, TopicRouteData topicRouteData) { + if (isTopicRouteValid(topicRouteData)) { +- MessageQueueView tmp = new MessageQueueView(topic, topicRouteData); ++ MessageQueueView tmp = new MessageQueueView(topic, topicRouteData, TopicRouteService.this.getMqFaultStrategy()); + log.debug("load topic route from namesrv. topic: {}, queue: {}", topic, tmp); + return tmp; + } +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java +index 7fd9a9ffd..77ae5e4d1 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java +@@ -93,7 +93,6 @@ public class ReceiveMessageActivityTest extends BaseActivityTest { + pollTimeCaptor.capture(), anyInt(), any(), anyBoolean(), any(), isNull(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(new PopResult(PopStatus.NO_NEW_MSG, Collections.emptyList()))); + +- + ProxyContext context = createContext(); + context.setRemainingMs(1L); + this.receiveMessageActivity.receiveMessage( +@@ -274,7 +273,7 @@ public class ReceiveMessageActivityTest extends BaseActivityTest { + } + + @Test +- public void testReceiveMessageQueueSelector() { ++ public void testReceiveMessageQueueSelector() throws Exception { + TopicRouteData topicRouteData = new TopicRouteData(); + List queueDatas = new ArrayList<>(); + for (int i = 0; i < 2; i++) { +@@ -298,7 +297,7 @@ public class ReceiveMessageActivityTest extends BaseActivityTest { + } + topicRouteData.setBrokerDatas(brokerDatas); + +- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); ++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); + ReceiveMessageActivity.ReceiveMessageQueueSelector selector = new ReceiveMessageActivity.ReceiveMessageQueueSelector(""); + + AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java +index 588423bb9..4882a5ed8 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java +@@ -35,6 +35,8 @@ import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ExecutionException; + import java.util.concurrent.TimeUnit; + import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.client.ClientConfig; ++import org.apache.rocketmq.client.latency.MQFaultStrategy; + import org.apache.rocketmq.client.producer.SendResult; + import org.apache.rocketmq.client.producer.SendStatus; + import org.apache.rocketmq.common.MixAll; +@@ -49,6 +51,7 @@ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; + import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; + import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; + import org.apache.rocketmq.proxy.service.route.MessageQueueView; ++import org.apache.rocketmq.proxy.service.route.TopicRouteService; + import org.apache.rocketmq.remoting.protocol.route.BrokerData; + import org.apache.rocketmq.remoting.protocol.route.QueueData; + import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +@@ -62,15 +65,19 @@ import static org.junit.Assert.assertThrows; + import static org.mockito.ArgumentMatchers.any; + import static org.mockito.ArgumentMatchers.anyInt; + import static org.mockito.ArgumentMatchers.anyString; ++import static org.mockito.Mockito.mock; + import static org.mockito.Mockito.when; + + public class SendMessageActivityTest extends BaseActivityTest { + + protected static final String BROKER_NAME = "broker"; ++ protected static final String BROKER_NAME2 = "broker2"; + protected static final String CLUSTER_NAME = "cluster"; + protected static final String BROKER_ADDR = "127.0.0.1:10911"; ++ protected static final String BROKER_ADDR2 = "127.0.0.1:10912"; + private static final String TOPIC = "topic"; + private static final String CONSUMER_GROUP = "consumerGroup"; ++ MQFaultStrategy mqFaultStrategy; + + private SendMessageActivity sendMessageActivity; + +@@ -262,7 +269,7 @@ public class SendMessageActivityTest extends BaseActivityTest { + } + + @Test +- public void testSendOrderMessageQueueSelector() { ++ public void testSendOrderMessageQueueSelector() throws Exception { + TopicRouteData topicRouteData = new TopicRouteData(); + QueueData queueData = new QueueData(); + BrokerData brokerData = new BrokerData(); +@@ -277,7 +284,7 @@ public class SendMessageActivityTest extends BaseActivityTest { + brokerData.setBrokerAddrs(brokerAddrs); + topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); + +- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); ++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); + SendMessageActivity.SendMessageQueueSelector selector1 = new SendMessageActivity.SendMessageQueueSelector( + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() +@@ -288,6 +295,12 @@ public class SendMessageActivityTest extends BaseActivityTest { + .build() + ); + ++ TopicRouteService topicRouteService = mock(TopicRouteService.class); ++ MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class); ++ when(topicRouteService.getAllMessageQueueView(any(), any())).thenReturn(messageQueueView); ++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); ++ when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false); ++ + SendMessageActivity.SendMessageQueueSelector selector2 = new SendMessageActivity.SendMessageQueueSelector( + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() +@@ -328,12 +341,17 @@ public class SendMessageActivityTest extends BaseActivityTest { + brokerData.setBrokerAddrs(brokerAddrs); + topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); + +- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); ++ + SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector( + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder().build()) + .build() + ); ++ TopicRouteService topicRouteService = mock(TopicRouteService.class); ++ MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class); ++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); ++ when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false); ++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy()); + + AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); + AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); +@@ -343,6 +361,45 @@ public class SendMessageActivityTest extends BaseActivityTest { + assertNotEquals(firstSelect, secondSelect); + } + ++ @Test ++ public void testSendNormalMessageQueueSelectorPipeLine() throws Exception { ++ TopicRouteData topicRouteData = new TopicRouteData(); ++ int queueNums = 2; ++ ++ QueueData queueData = createQueueData(BROKER_NAME, queueNums); ++ QueueData queueData2 = createQueueData(BROKER_NAME2, queueNums); ++ topicRouteData.setQueueDatas(Lists.newArrayList(queueData,queueData2)); ++ ++ ++ BrokerData brokerData = createBrokerData(CLUSTER_NAME, BROKER_NAME, BROKER_ADDR); ++ BrokerData brokerData2 = createBrokerData(CLUSTER_NAME, BROKER_NAME2, BROKER_ADDR2); ++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData, brokerData2)); ++ ++ SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector( ++ SendMessageRequest.newBuilder() ++ .addMessages(Message.newBuilder().build()) ++ .build() ++ ); ++ ++ ClientConfig cc = new ClientConfig(); ++ this.mqFaultStrategy = new MQFaultStrategy(cc, null, null); ++ mqFaultStrategy.setSendLatencyFaultEnable(true); ++ mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, true); ++ mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, false); ++ ++ TopicRouteService topicRouteService = mock(TopicRouteService.class); ++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); ++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy()); ++ ++ ++ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); ++ assertEquals(firstSelect.getBrokerName(), BROKER_NAME2); ++ ++ mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, false); ++ mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, true); ++ AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); ++ assertEquals(secondSelect.getBrokerName(), BROKER_NAME); ++ } + @Test + public void testParameterValidate() { + // too large message body +@@ -850,4 +907,23 @@ public class SendMessageActivityTest extends BaseActivityTest { + } + return sb.toString(); + } ++ ++ private static QueueData createQueueData(String brokerName, int writeQueueNums) { ++ QueueData queueData = new QueueData(); ++ queueData.setBrokerName(brokerName); ++ queueData.setWriteQueueNums(writeQueueNums); ++ queueData.setPerm(PermName.PERM_WRITE); ++ return queueData; ++ } ++ ++ private static BrokerData createBrokerData(String clusterName, String brokerName, String brokerAddrs) { ++ BrokerData brokerData = new BrokerData(); ++ brokerData.setCluster(clusterName); ++ brokerData.setBrokerName(brokerName); ++ HashMap brokerAddrsMap = new HashMap<>(); ++ brokerAddrsMap.put(MixAll.MASTER_ID, brokerAddrs); ++ brokerData.setBrokerAddrs(brokerAddrsMap); ++ ++ return brokerData; ++ } + } +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java +index c97bd5a72..ca6fe909e 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java +@@ -78,7 +78,7 @@ public class BaseServiceTest extends InitConfigTest { + topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); + + when(this.topicRouteService.getAllMessageQueueView(any(), eq(ERR_TOPIC))).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "")); +- when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData)); +- when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData)); ++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null)); ++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData, null)); + } + } +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java +index e44ed28f4..d150f87c4 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java +@@ -30,12 +30,12 @@ public class MessageQueueSelectorTest extends BaseServiceTest { + public void testReadMessageQueue() { + queueData.setPerm(PermName.PERM_READ); + queueData.setReadQueueNums(0); +- MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); ++ MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true); + assertTrue(messageQueueSelector.getQueues().isEmpty()); + + queueData.setPerm(PermName.PERM_READ); + queueData.setReadQueueNums(3); +- messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); ++ messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true); + assertEquals(3, messageQueueSelector.getQueues().size()); + assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); + for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { +@@ -58,12 +58,12 @@ public class MessageQueueSelectorTest extends BaseServiceTest { + public void testWriteMessageQueue() { + queueData.setPerm(PermName.PERM_WRITE); + queueData.setReadQueueNums(0); +- MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); ++ MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false); + assertTrue(messageQueueSelector.getQueues().isEmpty()); + + queueData.setPerm(PermName.PERM_WRITE); + queueData.setWriteQueueNums(3); +- messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); ++ messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false); + assertEquals(3, messageQueueSelector.getQueues().size()); + assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); + for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +index c67f4953d..43fba3d03 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +@@ -132,7 +132,7 @@ public class HeartbeatSyncerTest extends InitConfigTest { + brokerAddr.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddr); + topicRouteData.getBrokerDatas().add(brokerData); +- MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData); ++ MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData, null); + when(this.topicRouteService.getAllMessageQueueView(any(), anyString())).thenReturn(messageQueueView); + } + } +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java +index a0063544e..91af74cbe 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java +@@ -64,7 +64,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { + this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager, + this.mqClientAPIFactory); + +- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); ++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); + when(this.topicRouteService.getAllMessageQueueView(any(), anyString())) + .thenReturn(messageQueueView); + +@@ -127,7 +127,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { + brokerData.setBrokerAddrs(brokerAddrs); + topicRouteData.getQueueDatas().add(queueData); + topicRouteData.getBrokerDatas().add(brokerData); +- when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData)); ++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null)); + + TopicRouteData clusterTopicRouteData = new TopicRouteData(); + QueueData clusterQueueData = new QueueData(); +@@ -141,7 +141,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { + brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR); + clusterBrokerData.setBrokerAddrs(brokerAddrs); + clusterTopicRouteData.setBrokerDatas(Lists.newArrayList(clusterBrokerData)); +- when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData)); ++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData, null)); + + TopicRouteData clusterTopicRouteData2 = new TopicRouteData(); + QueueData clusterQueueData2 = new QueueData(); +@@ -155,7 +155,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { + brokerAddrs.put(MixAll.MASTER_ID, brokerAddr2); + clusterBrokerData2.setBrokerAddrs(brokerAddrs); + clusterTopicRouteData2.setBrokerDatas(Lists.newArrayList(clusterBrokerData2)); +- when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2)); ++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2, null)); + + ConfigurationManager.getProxyConfig().setTransactionHeartbeatBatchNum(2); + this.clusterTransactionService.start(); +-- +2.32.0.windows.2 + diff --git a/patch015-backport-fix-some-bugs.patch b/patch015-backport-fix-some-bugs.patch new file mode 100644 index 000000000..11c10a6bd --- /dev/null +++ b/patch015-backport-fix-some-bugs.patch @@ -0,0 +1,1894 @@ +From bd0e9c09db9748f7f74a0c707579142dccf30afc Mon Sep 17 00:00:00 2001 +From: PiteXChen <44110731+RapperCL@users.noreply.github.com> +Date: Tue, 29 Aug 2023 19:39:27 +0800 +Subject: [PATCH 1/7] [ISSUE #7111] Remove responseFuture from the + responseTable when exception occurs (#7112) + +* remove responseFuture when exception +* Empty-Commit + +--------- +Co-authored-by: chenyong152 +--- + .../apache/rocketmq/remoting/netty/NettyRemotingAbstract.java | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +index 44d6a3df4..fce2de267 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +@@ -529,6 +529,7 @@ public abstract class NettyRemotingAbstract { + log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); + }); + } catch (Exception e) { ++ responseTable.remove(opaque); + responseFuture.release(); + log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); + throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); +-- +2.32.0.windows.2 + + +From c78061bf6ca5f35452510ec4107c46735c51c316 Mon Sep 17 00:00:00 2001 +From: lizhimins <707364882@qq.com> +Date: Wed, 30 Aug 2023 22:29:51 +0800 +Subject: [PATCH 2/7] [ISSUE#7280] Fix and refactor handle commit exception in + tiered storage (#7281) + +* refactor handle commit exception + +* refactor handle commit exception + +* fix handle commit exception +--- + .../tieredstore/TieredDispatcher.java | 3 +- + .../tieredstore/TieredMessageFetcher.java | 57 ++-- + .../tieredstore/TieredMessageStore.java | 26 +- + .../provider/TieredFileSegment.java | 291 ++++++++++-------- + .../provider/TieredStoreProvider.java | 8 +- + .../provider/posix/PosixFileSegment.java | 4 +- + .../CommitLogInputStream.java} | 30 +- + .../FileSegmentInputStream.java} | 49 ++- + .../FileSegmentInputStreamFactory.java} | 26 +- + .../tieredstore/TieredMessageStoreTest.java | 14 +- + .../tieredstore/file/TieredFlatFileTest.java | 2 + + .../tieredstore/file/TieredIndexFileTest.java | 2 + + ...m.java => MockFileSegmentInputStream.java} | 8 +- + .../TieredFileSegmentInputStreamTest.java | 24 +- + .../provider/TieredFileSegmentTest.java | 89 +++++- + .../provider/memory/MemoryFileSegment.java | 27 +- + .../memory/MemoryFileSegmentWithoutCheck.java | 4 +- + 17 files changed, 427 insertions(+), 237 deletions(-) + rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredCommitLogInputStream.java => stream/CommitLogInputStream.java} (88%) + rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredFileSegmentInputStream.java => stream/FileSegmentInputStream.java} (77%) + rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredFileSegmentInputStreamFactory.java => stream/FileSegmentInputStreamFactory.java} (54%) + rename tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/{MockTieredFileSegmentInputStream.java => MockFileSegmentInputStream.java} (82%) + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +index 1746190cd..430c2b62e 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +@@ -318,8 +318,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch + continue; + case FILE_CLOSED: + tieredFlatFileManager.destroyCompositeFile(flatFile.getMessageQueue()); +- logger.info("TieredDispatcher#dispatchFlatFile: file has been close and destroy, " + +- "topic: {}, queueId: {}", topic, queueId); ++ logger.info("File has been closed and destroy, topic: {}, queueId: {}", topic, queueId); + return; + default: + dispatchOffset--; +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java +index 9a9a3e5a5..766ff64f6 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java +@@ -273,15 +273,17 @@ public class TieredMessageFetcher implements MessageStoreFetcher { + TieredStoreMetricsManager.cacheHit.add(resultWrapperList.size(), attributes); + } + +- // if no cached message found and there is currently an inflight request, wait for the request to end before continuing ++ // If there are no messages in the cache and there are currently requests being pulled. ++ // We need to wait for the request to return before continuing. + if (resultWrapperList.isEmpty() && waitInflightRequest) { +- CompletableFuture future = flatFile.getInflightRequest(group, queueOffset, maxCount) +- .getFuture(queueOffset); ++ CompletableFuture future = ++ flatFile.getInflightRequest(group, queueOffset, maxCount).getFuture(queueOffset); + if (!future.isDone()) { + Stopwatch stopwatch = Stopwatch.createStarted(); + // to prevent starvation issues, only allow waiting for inflight request once + return future.thenCompose(v -> { +- LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: wait for inflight request cost: {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); ++ LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: wait for response cost: {}ms", ++ stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, false); + }); + } +@@ -302,7 +304,8 @@ public class TieredMessageFetcher implements MessageStoreFetcher { + + // if cache hit, result will be returned immediately and asynchronously prefetch messages for later requests + if (!resultWrapperList.isEmpty()) { +- LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: cache hit: topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}", ++ LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: cache hit: " + ++ "topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}", + mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, resultWrapperList.size()); + prefetchMessage(flatFile, group, maxCount, lastGetOffset + 1); + +@@ -316,8 +319,10 @@ public class TieredMessageFetcher implements MessageStoreFetcher { + } + + // if cache is miss, immediately pull messages +- LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: topic: {}, queue: {}, queue offset: {}, max message num: {}", ++ LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + ++ "topic: {}, queue: {}, queue offset: {}, max message num: {}", + mq.getTopic(), mq.getQueueId(), queueOffset, maxCount); ++ + CompletableFuture resultFuture; + synchronized (flatFile) { + int batchSize = maxCount * storeConfig.getReadAheadMinFactor(); +@@ -453,42 +458,42 @@ public class TieredMessageFetcher implements MessageStoreFetcher { + public CompletableFuture getMessageAsync( + String group, String topic, int queueId, long queueOffset, int maxCount, final MessageFilter messageFilter) { + ++ GetMessageResult result = new GetMessageResult(); + CompositeQueueFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId)); ++ + if (flatFile == null) { +- GetMessageResult result = new GetMessageResult(); + result.setNextBeginOffset(queueOffset); + result.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE); + return CompletableFuture.completedFuture(result); + } + +- GetMessageResult result = new GetMessageResult(); +- long minQueueOffset = flatFile.getConsumeQueueMinOffset(); +- long maxQueueOffset = flatFile.getConsumeQueueCommitOffset(); +- result.setMinOffset(minQueueOffset); +- result.setMaxOffset(maxQueueOffset); ++ // Max queue offset means next message put position ++ result.setMinOffset(flatFile.getConsumeQueueMinOffset()); ++ result.setMaxOffset(flatFile.getConsumeQueueCommitOffset()); ++ ++ // Fill result according file offset. ++ // Offset range | Result | Fix to ++ // (-oo, 0] | no message | current offset ++ // (0, min) | too small | min offset ++ // [min, max) | correct | ++ // [max, max] | overflow one | max offset ++ // (max, +oo) | overflow badly | max offset + +- if (flatFile.getConsumeQueueCommitOffset() <= 0) { ++ if (result.getMaxOffset() <= 0) { + result.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE); + result.setNextBeginOffset(queueOffset); + return CompletableFuture.completedFuture(result); +- } +- +- // request range | result +- // (0, min) | too small +- // [min, max) | correct +- // [max, max] | overflow one +- // (max, +oo) | overflow badly +- if (queueOffset < minQueueOffset) { ++ } else if (queueOffset < result.getMinOffset()) { + result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL); +- result.setNextBeginOffset(flatFile.getConsumeQueueMinOffset()); ++ result.setNextBeginOffset(result.getMinOffset()); + return CompletableFuture.completedFuture(result); +- } else if (queueOffset == maxQueueOffset) { ++ } else if (queueOffset == result.getMaxOffset()) { + result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE); +- result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset()); ++ result.setNextBeginOffset(result.getMaxOffset()); + return CompletableFuture.completedFuture(result); +- } else if (queueOffset > maxQueueOffset) { ++ } else if (queueOffset > result.getMaxOffset()) { + result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY); +- result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset()); ++ result.setNextBeginOffset(result.getMaxOffset()); + return CompletableFuture.completedFuture(result); + } + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +index 5240ac8e9..78e855f36 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +@@ -99,11 +99,11 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + return storeConfig; + } + +- public boolean viaTieredStorage(String topic, int queueId, long offset) { +- return viaTieredStorage(topic, queueId, offset, 1); ++ public boolean fetchFromCurrentStore(String topic, int queueId, long offset) { ++ return fetchFromCurrentStore(topic, queueId, offset, 1); + } + +- public boolean viaTieredStorage(String topic, int queueId, long offset, int batchSize) { ++ public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int batchSize) { + TieredMessageStoreConfig.TieredStorageLevel deepStorageLevel = storeConfig.getTieredStorageLevel(); + + if (deepStorageLevel.check(TieredMessageStoreConfig.TieredStorageLevel.FORCE)) { +@@ -146,8 +146,10 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + public CompletableFuture getMessageAsync(String group, String topic, + int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { + +- if (!viaTieredStorage(topic, queueId, offset, maxMsgNums)) { +- logger.trace("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); ++ if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) { ++ logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); ++ } else { ++ logger.trace("GetMessageAsync from next store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); + return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); + } + +@@ -168,14 +170,14 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + + if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) { + TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes); +- logger.debug("GetMessageAsync not found then try back to next store, result: {}, " + ++ logger.debug("GetMessageAsync not found, then back to next store, result: {}, " + + "topic: {}, queue: {}, queue offset: {}, offset range: {}-{}", + result.getStatus(), topic, queueId, offset, result.getMinOffset(), result.getMaxOffset()); + return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); + } + } + +- // system topic ++ // Fetch system topic data from the broker when using the force level. + if (result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { + if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { + return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); +@@ -198,7 +200,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + TieredStoreMetricsManager.messagesOutTotal.add(result.getMessageCount(), messagesOutAttributes); + } + +- // fix min or max offset according next store ++ // Fix min or max offset according next store at last + long minOffsetInQueue = next.getMinOffsetInQueue(topic, queueId); + if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) { + result.setMinOffset(minOffsetInQueue); +@@ -209,7 +211,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + } + return result; + }).exceptionally(e -> { +- logger.error("GetMessageAsync from tiered store failed: ", e); ++ logger.error("GetMessageAsync from tiered store failed", e); + return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); + }); + } +@@ -251,7 +253,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + .build(); + TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); + if (time < 0) { +- logger.debug("TieredMessageStore#getEarliestMessageTimeAsync: get earliest message time failed, try to get earliest message time from next store: topic: {}, queue: {}", ++ logger.debug("GetEarliestMessageTimeAsync failed, try to get earliest message time from next store: topic: {}, queue: {}", + topic, queueId); + return finalNextEarliestMessageTime != Long.MAX_VALUE ? finalNextEarliestMessageTime : -1; + } +@@ -262,7 +264,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + @Override + public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, + long consumeQueueOffset) { +- if (viaTieredStorage(topic, queueId, consumeQueueOffset)) { ++ if (fetchFromCurrentStore(topic, queueId, consumeQueueOffset)) { + Stopwatch stopwatch = Stopwatch.createStarted(); + return fetcher.getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset) + .thenApply(time -> { +@@ -272,7 +274,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + .build(); + TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); + if (time == -1) { +- logger.debug("TieredMessageStore#getMessageStoreTimeStampAsync: get message time failed, try to get message time from next store: topic: {}, queue: {}, queue offset: {}", ++ logger.debug("GetEarliestMessageTimeAsync failed, try to get message time from next store, topic: {}, queue: {}, queue offset: {}", + topic, queueId, consumeQueueOffset); + return next.getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset); + } +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java +index 5062c7d9e..32911a6e8 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java +@@ -16,14 +16,11 @@ + */ + package org.apache.rocketmq.tieredstore.provider; + +-import com.google.common.annotations.VisibleForTesting; +-import com.google.common.base.Stopwatch; + import java.nio.ByteBuffer; + import java.util.ArrayList; + import java.util.List; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.Semaphore; +-import java.util.concurrent.TimeUnit; + import java.util.concurrent.locks.ReentrantLock; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +@@ -35,8 +32,8 @@ import org.apache.rocketmq.tieredstore.exception.TieredStoreException; + import org.apache.rocketmq.tieredstore.file.TieredCommitLog; + import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue; + import org.apache.rocketmq.tieredstore.file.TieredIndexFile; +-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory; ++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory; + import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; + import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + +@@ -50,22 +47,23 @@ public abstract class TieredFileSegment implements Comparable + protected final TieredMessageStoreConfig storeConfig; + + private final long maxSize; +- private final ReentrantLock bufferLock; +- private final Semaphore commitLock; ++ private final ReentrantLock bufferLock = new ReentrantLock(); ++ private final Semaphore commitLock = new Semaphore(1); + +- private volatile boolean full; +- private volatile boolean closed; ++ private volatile boolean full = false; ++ private volatile boolean closed = false; + +- private volatile long minTimestamp; +- private volatile long maxTimestamp; +- private volatile long commitPosition; +- private volatile long appendPosition; ++ private volatile long minTimestamp = Long.MAX_VALUE; ++ private volatile long maxTimestamp = Long.MAX_VALUE; ++ private volatile long commitPosition = 0L; ++ private volatile long appendPosition = 0L; + + // only used in commitLog +- private volatile long dispatchCommitOffset = 0; ++ private volatile long dispatchCommitOffset = 0L; + + private ByteBuffer codaBuffer; +- private List uploadBufferList = new ArrayList<>(); ++ private List bufferList = new ArrayList<>(); ++ private FileSegmentInputStream fileSegmentInputStream; + private CompletableFuture flightCommitRequest = CompletableFuture.completedFuture(false); + + public TieredFileSegment(TieredMessageStoreConfig storeConfig, +@@ -75,21 +73,13 @@ public abstract class TieredFileSegment implements Comparable + this.fileType = fileType; + this.filePath = filePath; + this.baseOffset = baseOffset; +- +- this.closed = false; +- this.bufferLock = new ReentrantLock(); +- this.commitLock = new Semaphore(1); +- +- this.commitPosition = 0L; +- this.appendPosition = 0L; +- this.minTimestamp = Long.MAX_VALUE; +- this.maxTimestamp = Long.MAX_VALUE; +- +- // The max segment size of a file is determined by the file type +- this.maxSize = getMaxSizeAccordingFileType(storeConfig); ++ this.maxSize = getMaxSizeByFileType(); + } + +- private long getMaxSizeAccordingFileType(TieredMessageStoreConfig storeConfig) { ++ /** ++ * The max segment size of a file is determined by the file type ++ */ ++ protected long getMaxSizeByFileType() { + switch (fileType) { + case COMMIT_LOG: + return storeConfig.getTieredStoreCommitLogMaxSize(); +@@ -184,39 +174,23 @@ public abstract class TieredFileSegment implements Comparable + this.appendPosition = pos; + } + +- private List rollingUploadBuffer() { ++ private List borrowBuffer() { + bufferLock.lock(); + try { +- List tmp = uploadBufferList; +- uploadBufferList = new ArrayList<>(); ++ List tmp = bufferList; ++ bufferList = new ArrayList<>(); + return tmp; + } finally { + bufferLock.unlock(); + } + } + +- private void sendBackBuffer(TieredFileSegmentInputStream inputStream) { +- bufferLock.lock(); +- try { +- List tmpBufferList = inputStream.getUploadBufferList(); +- for (ByteBuffer buffer : tmpBufferList) { +- buffer.rewind(); +- } +- tmpBufferList.addAll(uploadBufferList); +- uploadBufferList = tmpBufferList; +- if (inputStream.getCodaBuffer() != null) { +- codaBuffer.rewind(); +- } +- } finally { +- bufferLock.unlock(); +- } +- } +- + @SuppressWarnings("NonAtomicOperationOnVolatileField") +- public AppendResult append(ByteBuffer byteBuf, long timeStamp) { ++ public AppendResult append(ByteBuffer byteBuf, long timestamp) { + if (closed) { + return AppendResult.FILE_CLOSED; + } ++ + bufferLock.lock(); + try { + if (full || codaBuffer != null) { +@@ -227,7 +201,8 @@ public abstract class TieredFileSegment implements Comparable + minTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_BEGIN_TIME_STAMP_POSITION); + maxTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_END_TIME_STAMP_POSITION); + appendPosition += byteBuf.remaining(); +- uploadBufferList.add(byteBuf); ++ // IndexFile is large and not change after compaction, no need deep copy ++ bufferList.add(byteBuf); + setFull(); + return AppendResult.SUCCESS; + } +@@ -236,23 +211,34 @@ public abstract class TieredFileSegment implements Comparable + setFull(); + return AppendResult.FILE_FULL; + } +- if (uploadBufferList.size() > storeConfig.getTieredStoreGroupCommitCount() ++ ++ if (bufferList.size() > storeConfig.getTieredStoreGroupCommitCount() + || appendPosition - commitPosition > storeConfig.getTieredStoreGroupCommitSize()) { + commitAsync(); + } +- if (uploadBufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) { +- logger.debug("TieredFileSegment#append: buffer full: file: {}, upload buffer size: {}", +- getPath(), uploadBufferList.size()); ++ ++ if (bufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) { ++ logger.debug("File segment append buffer full, file: {}, buffer size: {}, pending bytes: {}", ++ getPath(), bufferList.size(), appendPosition - commitPosition); + return AppendResult.BUFFER_FULL; + } +- if (timeStamp != Long.MAX_VALUE) { +- maxTimestamp = timeStamp; ++ ++ if (timestamp != Long.MAX_VALUE) { ++ maxTimestamp = timestamp; + if (minTimestamp == Long.MAX_VALUE) { +- minTimestamp = timeStamp; ++ minTimestamp = timestamp; + } + } ++ + appendPosition += byteBuf.remaining(); +- uploadBufferList.add(byteBuf); ++ ++ // deep copy buffer ++ ByteBuffer byteBuffer = ByteBuffer.allocateDirect(byteBuf.remaining()); ++ byteBuffer.put(byteBuf); ++ byteBuffer.flip(); ++ byteBuf.rewind(); ++ ++ bufferList.add(byteBuffer); + return AppendResult.SUCCESS; + } finally { + bufferLock.unlock(); +@@ -267,7 +253,6 @@ public abstract class TieredFileSegment implements Comparable + return appendPosition; + } + +- @VisibleForTesting + public void setAppendPosition(long appendPosition) { + this.appendPosition = appendPosition; + } +@@ -333,6 +318,8 @@ public abstract class TieredFileSegment implements Comparable + if (closed) { + return false; + } ++ // result is false when we send real commit request ++ // use join for wait flight request done + Boolean result = commitAsync().join(); + if (!result) { + result = flightCommitRequest.join(); +@@ -340,92 +327,156 @@ public abstract class TieredFileSegment implements Comparable + return result; + } + ++ private void releaseCommitLock() { ++ if (commitLock.availablePermits() == 0) { ++ commitLock.release(); ++ } else { ++ logger.error("[Bug] FileSegmentCommitAsync, lock is already released: available permits: {}", ++ commitLock.availablePermits()); ++ } ++ } ++ ++ private void updateDispatchCommitOffset(List bufferList) { ++ if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) { ++ dispatchCommitOffset = ++ MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1)); ++ } ++ } ++ ++ /** ++ * @return false: commit, true: no commit operation ++ */ + @SuppressWarnings("NonAtomicOperationOnVolatileField") + public CompletableFuture commitAsync() { + if (closed) { + return CompletableFuture.completedFuture(false); + } +- Stopwatch stopwatch = Stopwatch.createStarted(); ++ + if (!needCommit()) { + return CompletableFuture.completedFuture(true); + } +- try { +- int permits = commitLock.drainPermits(); +- if (permits <= 0) { +- return CompletableFuture.completedFuture(false); +- } +- } catch (Exception e) { ++ ++ if (commitLock.drainPermits() <= 0) { + return CompletableFuture.completedFuture(false); + } +- List bufferList = rollingUploadBuffer(); +- int bufferSize = 0; +- for (ByteBuffer buffer : bufferList) { +- bufferSize += buffer.remaining(); +- } +- if (codaBuffer != null) { +- bufferSize += codaBuffer.remaining(); +- } +- if (bufferSize == 0) { +- return CompletableFuture.completedFuture(true); +- } +- TieredFileSegmentInputStream inputStream = TieredFileSegmentInputStreamFactory.build( +- fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize); +- int finalBufferSize = bufferSize; ++ + try { +- flightCommitRequest = commit0(inputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX) ++ if (fileSegmentInputStream != null) { ++ long fileSize = this.getSize(); ++ if (fileSize == -1L) { ++ logger.error("Get commit position error before commit, Commit: %d, Expect: %d, Current Max: %d, FileName: %s", ++ commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath()); ++ releaseCommitLock(); ++ return CompletableFuture.completedFuture(false); ++ } else { ++ if (correctPosition(fileSize, null)) { ++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); ++ fileSegmentInputStream = null; ++ } ++ } ++ } ++ ++ int bufferSize; ++ if (fileSegmentInputStream != null) { ++ bufferSize = fileSegmentInputStream.available(); ++ } else { ++ List bufferList = borrowBuffer(); ++ bufferSize = bufferList.stream().mapToInt(ByteBuffer::remaining).sum() ++ + (codaBuffer != null ? codaBuffer.remaining() : 0); ++ if (bufferSize == 0) { ++ releaseCommitLock(); ++ return CompletableFuture.completedFuture(true); ++ } ++ fileSegmentInputStream = FileSegmentInputStreamFactory.build( ++ fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize); ++ } ++ ++ return flightCommitRequest = this ++ .commit0(fileSegmentInputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX) + .thenApply(result -> { + if (result) { +- if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) { +- dispatchCommitOffset = MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1)); +- } +- commitPosition += finalBufferSize; ++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); ++ commitPosition += bufferSize; ++ fileSegmentInputStream = null; + return true; +- } +- sendBackBuffer(inputStream); +- return false; +- }) +- .exceptionally(e -> handleCommitException(inputStream, e)) +- .whenComplete((result, e) -> { +- if (commitLock.availablePermits() == 0) { +- logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize); +- commitLock.release(); + } else { +- logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits()); ++ fileSegmentInputStream.rewind(); ++ return false; + } +- }); +- return flightCommitRequest; ++ }) ++ .exceptionally(this::handleCommitException) ++ .whenComplete((result, e) -> releaseCommitLock()); ++ + } catch (Exception e) { +- handleCommitException(inputStream, e); +- if (commitLock.availablePermits() == 0) { +- logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize); +- commitLock.release(); +- } else { +- logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits()); +- } ++ handleCommitException(e); ++ releaseCommitLock(); + } + return CompletableFuture.completedFuture(false); + } + +- private boolean handleCommitException(TieredFileSegmentInputStream inputStream, Throwable e) { ++ private long getCorrectFileSize(Throwable throwable) { ++ if (throwable instanceof TieredStoreException) { ++ long fileSize = ((TieredStoreException) throwable).getPosition(); ++ if (fileSize > 0) { ++ return fileSize; ++ } ++ } ++ return getSize(); ++ } ++ ++ private boolean handleCommitException(Throwable e) { ++ // Get root cause here + Throwable cause = e.getCause() != null ? e.getCause() : e; +- sendBackBuffer(inputStream); +- long realSize = 0; +- if (cause instanceof TieredStoreException && ((TieredStoreException) cause).getPosition() > 0) { +- realSize = ((TieredStoreException) cause).getPosition(); ++ long fileSize = this.getCorrectFileSize(cause); ++ ++ if (fileSize == -1L) { ++ logger.error("Get commit position error, Commit: %d, Expect: %d, Current Max: %d, FileName: %s", ++ commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath()); ++ fileSegmentInputStream.rewind(); ++ return false; ++ } ++ ++ if (correctPosition(fileSize, cause)) { ++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); ++ fileSegmentInputStream = null; ++ return true; ++ } else { ++ fileSegmentInputStream.rewind(); ++ return false; + } +- if (realSize <= 0) { +- realSize = getSize(); ++ } ++ ++ /** ++ * return true to clear buffer ++ */ ++ private boolean correctPosition(long fileSize, Throwable throwable) { ++ ++ // Current we have three offsets here: commit offset, expect offset, file size. ++ // We guarantee that the commit offset is less than or equal to the expect offset. ++ // Max offset will increase because we can continuously put in new buffers ++ String handleInfo = throwable == null ? "before commit" : "after commit"; ++ long expectPosition = commitPosition + fileSegmentInputStream.getContentLength(); ++ ++ String offsetInfo = String.format("Correct Commit Position, %s, result=[{}], " + ++ "Commit: %d, Expect: %d, Current Max: %d, FileSize: %d, FileName: %s", ++ handleInfo, commitPosition, expectPosition, appendPosition, fileSize, this.getPath()); ++ ++ // We are believing that the file size returned by the server is correct, ++ // can reset the commit offset to the file size reported by the storage system. ++ if (fileSize == expectPosition) { ++ logger.info(offsetInfo, "Success", throwable); ++ commitPosition = fileSize; ++ return true; + } +- if (realSize > 0 && realSize > commitPosition) { +- logger.error("TieredFileSegment#handleCommitException: commit failed: file: {}, try to fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause); +- // TODO check if this diff part is uploaded to backend storage +- long diff = appendPosition - commitPosition; +- commitPosition = realSize; +- appendPosition = realSize + diff; +- // TODO check if appendPosition is large than maxOffset +- } else if (realSize < commitPosition) { +- logger.error("[Bug]TieredFileSegment#handleCommitException: commit failed: file: {}, can not fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause); ++ ++ if (fileSize < commitPosition) { ++ logger.error(offsetInfo, "FileSizeIncorrect", throwable); ++ } else if (fileSize == commitPosition) { ++ logger.warn(offsetInfo, "CommitFailed", throwable); ++ } else if (fileSize > commitPosition) { ++ logger.warn(offsetInfo, "PartialSuccess", throwable); + } ++ commitPosition = fileSize; + return false; + } + } +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java +index 5a0ca25f5..0db3eaf8f 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java +@@ -18,7 +18,7 @@ package org.apache.rocketmq.tieredstore.provider; + + import java.nio.ByteBuffer; + import java.util.concurrent.CompletableFuture; +-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; ++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; + + public interface TieredStoreProvider { + +@@ -30,7 +30,9 @@ public interface TieredStoreProvider { + String getPath(); + + /** +- * Get file size in backend file system ++ * Get the real length of the file. ++ * Return 0 if the file does not exist, ++ * Return -1 if system get size failed. + * + * @return file real size + */ +@@ -71,5 +73,5 @@ public interface TieredStoreProvider { + * @param append try to append or create a new file + * @return put result, true if data successfully write; false otherwise + */ +- CompletableFuture commit0(TieredFileSegmentInputStream inputStream,long position, int length, boolean append); ++ CompletableFuture commit0(FileSegmentInputStream inputStream,long position, int length, boolean append); + } +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java +index 52be90b1d..7e949cb28 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java +@@ -36,7 +36,7 @@ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; + import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor; + import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; + import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; +-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; ++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; + import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + + import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_FILE_TYPE; +@@ -184,7 +184,7 @@ public class PosixFileSegment extends TieredFileSegment { + + @Override + public CompletableFuture commit0( +- TieredFileSegmentInputStream inputStream, long position, int length, boolean append) { ++ FileSegmentInputStream inputStream, long position, int length, boolean append) { + + Stopwatch stopwatch = Stopwatch.createStarted(); + AttributesBuilder attributesBuilder = newAttributesBuilder() +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java +similarity index 88% +rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java +rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java +index c70bb7656..13b6e0ef9 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java +@@ -15,7 +15,7 @@ + * limitations under the License. + */ + +-package org.apache.rocketmq.tieredstore.provider.inputstream; ++package org.apache.rocketmq.tieredstore.provider.stream; + + import java.io.IOException; + import java.nio.ByteBuffer; +@@ -23,20 +23,23 @@ import java.util.List; + import org.apache.rocketmq.tieredstore.common.FileSegmentType; + import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; + +-public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { ++public class CommitLogInputStream extends FileSegmentInputStream { + + /** + * commitLogOffset is the real physical offset of the commitLog buffer which is being read + */ ++ private final long startCommitLogOffset; ++ + private long commitLogOffset; + + private final ByteBuffer codaBuffer; + + private long markCommitLogOffset = -1; + +- public TieredCommitLogInputStream(FileSegmentType fileType, long startOffset, ++ public CommitLogInputStream(FileSegmentType fileType, long startOffset, + List uploadBufferList, ByteBuffer codaBuffer, int contentLength) { + super(fileType, uploadBufferList, contentLength); ++ this.startCommitLogOffset = startOffset; + this.commitLogOffset = startOffset; + this.codaBuffer = codaBuffer; + } +@@ -53,6 +56,15 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { + this.commitLogOffset = markCommitLogOffset; + } + ++ @Override ++ public synchronized void rewind() { ++ super.rewind(); ++ this.commitLogOffset = this.startCommitLogOffset; ++ if (this.codaBuffer != null) { ++ this.codaBuffer.rewind(); ++ } ++ } ++ + @Override + public ByteBuffer getCodaBuffer() { + return this.codaBuffer; +@@ -64,17 +76,17 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { + return -1; + } + readPosition++; +- if (curReadBufferIndex >= uploadBufferList.size()) { ++ if (curReadBufferIndex >= bufferList.size()) { + return readCoda(); + } + int res; + if (readPosInCurBuffer >= curBuffer.remaining()) { + curReadBufferIndex++; +- if (curReadBufferIndex >= uploadBufferList.size()) { ++ if (curReadBufferIndex >= bufferList.size()) { + readPosInCurBuffer = 0; + return readCoda(); + } +- curBuffer = uploadBufferList.get(curReadBufferIndex); ++ curBuffer = bufferList.get(curReadBufferIndex); + commitLogOffset += readPosInCurBuffer; + readPosInCurBuffer = 0; + } +@@ -119,9 +131,9 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { + int posInCurBuffer = readPosInCurBuffer; + long curCommitLogOffset = commitLogOffset; + ByteBuffer curBuf = curBuffer; +- while (needRead > 0 && bufIndex <= uploadBufferList.size()) { ++ while (needRead > 0 && bufIndex <= bufferList.size()) { + int readLen, remaining, realReadLen = 0; +- if (bufIndex == uploadBufferList.size()) { ++ if (bufIndex == bufferList.size()) { + // read from coda buffer + remaining = codaBuffer.remaining() - posInCurBuffer; + readLen = Math.min(remaining, needRead); +@@ -137,7 +149,7 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { + } + remaining = curBuf.remaining() - posInCurBuffer; + readLen = Math.min(remaining, needRead); +- curBuf = uploadBufferList.get(bufIndex); ++ curBuf = bufferList.get(bufIndex); + if (posInCurBuffer < MessageBufferUtil.PHYSICAL_OFFSET_POSITION) { + realReadLen = Math.min(MessageBufferUtil.PHYSICAL_OFFSET_POSITION - posInCurBuffer, readLen); + // read from commitLog buffer +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java +similarity index 77% +rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java +rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java +index e1758ca93..9e9d5135c 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java +@@ -15,15 +15,16 @@ + * limitations under the License. + */ + +-package org.apache.rocketmq.tieredstore.provider.inputstream; ++package org.apache.rocketmq.tieredstore.provider.stream; + + import java.io.IOException; + import java.io.InputStream; + import java.nio.ByteBuffer; + import java.util.List; ++import org.apache.commons.collections.CollectionUtils; + import org.apache.rocketmq.tieredstore.common.FileSegmentType; + +-public class TieredFileSegmentInputStream extends InputStream { ++public class FileSegmentInputStream extends InputStream { + + /** + * file type, can be commitlog, consume queue or indexfile now +@@ -33,7 +34,7 @@ public class TieredFileSegmentInputStream extends InputStream { + /** + * hold bytebuffer + */ +- protected final List uploadBufferList; ++ protected final List bufferList; + + /** + * total remaining of bytebuffer list +@@ -65,13 +66,13 @@ public class TieredFileSegmentInputStream extends InputStream { + + private int markReadPosInCurBuffer = -1; + +- public TieredFileSegmentInputStream(FileSegmentType fileType, List uploadBufferList, +- int contentLength) { ++ public FileSegmentInputStream( ++ FileSegmentType fileType, List bufferList, int contentLength) { + this.fileType = fileType; + this.contentLength = contentLength; +- this.uploadBufferList = uploadBufferList; +- if (uploadBufferList != null && uploadBufferList.size() > 0) { +- this.curBuffer = uploadBufferList.get(curReadBufferIndex); ++ this.bufferList = bufferList; ++ if (bufferList != null && bufferList.size() > 0) { ++ this.curBuffer = bufferList.get(curReadBufferIndex); + } + } + +@@ -95,18 +96,34 @@ public class TieredFileSegmentInputStream extends InputStream { + this.readPosition = markReadPosition; + this.curReadBufferIndex = markCurReadBufferIndex; + this.readPosInCurBuffer = markReadPosInCurBuffer; +- if (this.curReadBufferIndex < uploadBufferList.size()) { +- this.curBuffer = uploadBufferList.get(curReadBufferIndex); ++ if (this.curReadBufferIndex < bufferList.size()) { ++ this.curBuffer = bufferList.get(curReadBufferIndex); + } + } + ++ public synchronized void rewind() { ++ this.readPosition = 0; ++ this.curReadBufferIndex = 0; ++ this.readPosInCurBuffer = 0; ++ if (CollectionUtils.isNotEmpty(bufferList)) { ++ this.curBuffer = bufferList.get(0); ++ for (ByteBuffer buffer : bufferList) { ++ buffer.rewind(); ++ } ++ } ++ } ++ ++ public int getContentLength() { ++ return contentLength; ++ } ++ + @Override + public int available() { + return contentLength - readPosition; + } + +- public List getUploadBufferList() { +- return uploadBufferList; ++ public List getBufferList() { ++ return bufferList; + } + + public ByteBuffer getCodaBuffer() { +@@ -121,10 +138,10 @@ public class TieredFileSegmentInputStream extends InputStream { + readPosition++; + if (readPosInCurBuffer >= curBuffer.remaining()) { + curReadBufferIndex++; +- if (curReadBufferIndex >= uploadBufferList.size()) { ++ if (curReadBufferIndex >= bufferList.size()) { + return -1; + } +- curBuffer = uploadBufferList.get(curReadBufferIndex); ++ curBuffer = bufferList.get(curReadBufferIndex); + readPosInCurBuffer = 0; + } + return curBuffer.get(readPosInCurBuffer++) & 0xff; +@@ -153,8 +170,8 @@ public class TieredFileSegmentInputStream extends InputStream { + int bufIndex = curReadBufferIndex; + int posInCurBuffer = readPosInCurBuffer; + ByteBuffer curBuf = curBuffer; +- while (needRead > 0 && bufIndex < uploadBufferList.size()) { +- curBuf = uploadBufferList.get(bufIndex); ++ while (needRead > 0 && bufIndex < bufferList.size()) { ++ curBuf = bufferList.get(bufIndex); + int remaining = curBuf.remaining() - posInCurBuffer; + int readLen = Math.min(remaining, needRead); + // read from curBuf +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java +similarity index 54% +rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java +rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java +index d0c983fd4..a90baff3a 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java +@@ -15,30 +15,34 @@ + * limitations under the License. + */ + +-package org.apache.rocketmq.tieredstore.provider.inputstream; ++package org.apache.rocketmq.tieredstore.provider.stream; + + import java.nio.ByteBuffer; + import java.util.List; + import org.apache.rocketmq.tieredstore.common.FileSegmentType; + +-public class TieredFileSegmentInputStreamFactory { ++public class FileSegmentInputStreamFactory { + +- public static TieredFileSegmentInputStream build(FileSegmentType fileType, +- long startOffset, List uploadBufferList, ByteBuffer codaBuffer, int contentLength) { ++ public static FileSegmentInputStream build( ++ FileSegmentType fileType, long offset, List bufferList, ByteBuffer byteBuffer, int length) { ++ ++ if (bufferList == null) { ++ throw new IllegalArgumentException("bufferList is null"); ++ } + + switch (fileType) { + case COMMIT_LOG: +- return new TieredCommitLogInputStream( +- fileType, startOffset, uploadBufferList, codaBuffer, contentLength); ++ return new CommitLogInputStream( ++ fileType, offset, bufferList, byteBuffer, length); + case CONSUME_QUEUE: +- return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength); ++ return new FileSegmentInputStream(fileType, bufferList, length); + case INDEX: +- if (uploadBufferList.size() != 1) { +- throw new IllegalArgumentException("uploadBufferList size in INDEX type input stream must be 1"); ++ if (bufferList.size() != 1) { ++ throw new IllegalArgumentException("buffer block size must be 1 when file type is IndexFile"); + } +- return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength); ++ return new FileSegmentInputStream(fileType, bufferList, length); + default: +- throw new IllegalArgumentException("fileType is not supported"); ++ throw new IllegalArgumentException("file type is not supported"); + } + } + } +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java +index 8601392e7..2451199c2 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java +@@ -130,36 +130,36 @@ public class TieredMessageStoreTest { + // TieredStorageLevel.DISABLE + properties.setProperty("tieredStorageLevel", "0"); + configuration.update(properties); +- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); ++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); + + // TieredStorageLevel.NOT_IN_DISK + properties.setProperty("tieredStorageLevel", "1"); + configuration.update(properties); + when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false); +- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); ++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); + + when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); +- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); ++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); + + // TieredStorageLevel.NOT_IN_MEM + properties.setProperty("tieredStorageLevel", "2"); + configuration.update(properties); + Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false); + Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true); +- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); ++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); + + Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); + Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(false); +- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); ++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); + + Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); + Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true); +- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); ++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); + + // TieredStorageLevel.FORCE + properties.setProperty("tieredStorageLevel", "3"); + configuration.update(properties); +- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); ++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); + } + + @Test +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java +index cc39cfbfc..7a4d05969 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java +@@ -24,6 +24,7 @@ import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; + import org.apache.rocketmq.tieredstore.common.FileSegmentType; + import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; ++import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor; + import org.apache.rocketmq.tieredstore.metadata.FileSegmentMetadata; + import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore; + import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; +@@ -55,6 +56,7 @@ public class TieredFlatFileTest { + public void tearDown() throws IOException { + TieredStoreTestUtil.destroyMetadataStore(); + TieredStoreTestUtil.destroyTempDir(storePath); ++ TieredStoreExecutor.shutdown(); + } + + private List getSegmentMetadataList(TieredMetadataStore metadataStore) { +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +index 262d6645b..2da72bc7a 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +@@ -87,5 +87,7 @@ public class TieredIndexFileTest { + + indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); + Assert.assertEquals(1, indexList.size()); ++ ++ indexFile.destroy(); + } + } +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java +similarity index 82% +rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java +rename to tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java +index a6566b7de..3bbe41dd4 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java +@@ -20,13 +20,13 @@ package org.apache.rocketmq.tieredstore.provider; + import java.io.InputStream; + import java.nio.ByteBuffer; + import java.util.List; +-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; ++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; + +-public class MockTieredFileSegmentInputStream extends TieredFileSegmentInputStream { ++public class MockFileSegmentInputStream extends FileSegmentInputStream { + + private final InputStream inputStream; + +- public MockTieredFileSegmentInputStream(InputStream inputStream) { ++ public MockFileSegmentInputStream(InputStream inputStream) { + super(null, null, Integer.MAX_VALUE); + this.inputStream = inputStream; + } +@@ -43,7 +43,7 @@ public class MockTieredFileSegmentInputStream extends TieredFileSegmentInputStre + } + + @Override +- public List getUploadBufferList() { ++ public List getBufferList() { + return null; + } + +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java +index a2554ba3d..743d9182c 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java +@@ -28,8 +28,8 @@ import java.util.Random; + import org.apache.rocketmq.tieredstore.common.FileSegmentType; + import org.apache.rocketmq.tieredstore.file.TieredCommitLog; + import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue; +-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory; ++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory; + import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; + import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest; + import org.junit.Assert; +@@ -57,7 +57,7 @@ public class TieredFileSegmentInputStreamTest { + bufferSize += byteBuffer.remaining(); + } + +- // build expected byte buffer for verifying the TieredFileSegmentInputStream ++ // build expected byte buffer for verifying the FileSegmentInputStream + ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); + for (ByteBuffer byteBuffer : uploadBufferList) { + expectedByteBuffer.put(byteBuffer); +@@ -74,7 +74,7 @@ public class TieredFileSegmentInputStreamTest { + int[] batchReadSizeTestSet = { + MessageBufferUtil.PHYSICAL_OFFSET_POSITION - 1, MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtil.PHYSICAL_OFFSET_POSITION + 1, MSG_LEN - 1, MSG_LEN, MSG_LEN + 1 + }; +- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( ++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( + FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), finalBufferSize, batchReadSizeTestSet); + + } +@@ -98,7 +98,7 @@ public class TieredFileSegmentInputStreamTest { + int codaBufferSize = codaBuffer.remaining(); + bufferSize += codaBufferSize; + +- // build expected byte buffer for verifying the TieredFileSegmentInputStream ++ // build expected byte buffer for verifying the FileSegmentInputStream + ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); + for (ByteBuffer byteBuffer : uploadBufferList) { + expectedByteBuffer.put(byteBuffer); +@@ -119,7 +119,7 @@ public class TieredFileSegmentInputStreamTest { + MSG_LEN - 1, MSG_LEN, MSG_LEN + 1, + bufferSize - 1, bufferSize, bufferSize + 1 + }; +- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( ++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( + FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, codaBuffer, finalBufferSize), finalBufferSize, batchReadSizeTestSet); + + } +@@ -134,7 +134,7 @@ public class TieredFileSegmentInputStreamTest { + bufferSize += byteBuffer.remaining(); + } + +- // build expected byte buffer for verifying the TieredFileSegmentInputStream ++ // build expected byte buffer for verifying the FileSegmentInputStream + ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); + for (ByteBuffer byteBuffer : uploadBufferList) { + expectedByteBuffer.put(byteBuffer); +@@ -143,7 +143,7 @@ public class TieredFileSegmentInputStreamTest { + + int finalBufferSize = bufferSize; + int[] batchReadSizeTestSet = {TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE - 1, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE + 1}; +- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( ++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( + FileSegmentType.CONSUME_QUEUE, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), bufferSize, batchReadSizeTestSet); + } + +@@ -156,16 +156,16 @@ public class TieredFileSegmentInputStreamTest { + byteBuffer.flip(); + List uploadBufferList = Arrays.asList(byteBuffer); + +- // build expected byte buffer for verifying the TieredFileSegmentInputStream ++ // build expected byte buffer for verifying the FileSegmentInputStream + ByteBuffer expectedByteBuffer = byteBuffer.slice(); + +- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( ++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( + FileSegmentType.INDEX, COMMIT_LOG_START_OFFSET, uploadBufferList, null, byteBuffer.limit()), byteBuffer.limit(), new int[] {23, 24, 25}); + } + +- private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier constructor, ++ private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier constructor, + int bufferSize, int[] readBatchSizeTestSet) { +- TieredFileSegmentInputStream inputStream = constructor.get(); ++ FileSegmentInputStream inputStream = constructor.get(); + + // verify + verifyInputStream(inputStream, expectedByteBuffer); +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java +index 4cd83e0d2..a655710a5 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java +@@ -116,13 +116,22 @@ public class TieredFileSegmentTest { + } + + @Test +- public void testCommitFailed() { ++ public void testCommitFailedThenSuccess() { + long startTime = System.currentTimeMillis(); + MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG); + long lastSize = segment.getSize(); +- segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0); +- segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0); +- ++ segment.setCheckSize(false); ++ segment.initPosition(lastSize); ++ segment.setSize((int) lastSize); ++ ++ ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( ++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize); ++ ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( ++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN); ++ segment.append(buffer1, 0); ++ segment.append(buffer2, 0); ++ ++ // Mock new message arrive + segment.blocker = new CompletableFuture<>(); + new Thread(() -> { + try { +@@ -131,20 +140,88 @@ public class TieredFileSegmentTest { + Assert.fail(e.getMessage()); + } + ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer(); ++ buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2); + buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime); + segment.append(buffer, 0); + segment.blocker.complete(false); + }).start(); + ++ // Commit failed + segment.commit(); + segment.blocker.join(); ++ segment.blocker = null; ++ ++ // Copy data and assume commit success ++ segment.getMemStore().put(buffer1); ++ segment.getMemStore().put(buffer2); ++ segment.setSize((int) (lastSize + MessageBufferUtilTest.MSG_LEN * 2)); + +- segment.blocker = new CompletableFuture<>(); +- segment.blocker.complete(true); + segment.commit(); ++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition()); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset()); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); ++ ++ ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN); ++ Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1)); ++ ++ ByteBuffer msg2 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtil.getCommitLogOffset(msg2)); ++ ++ ByteBuffer msg3 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtilTest.MSG_LEN); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtil.getCommitLogOffset(msg3)); ++ } ++ ++ @Test ++ public void testCommitFailed3Times() { ++ long startTime = System.currentTimeMillis(); ++ MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG); ++ long lastSize = segment.getSize(); ++ segment.setCheckSize(false); ++ segment.initPosition(lastSize); ++ segment.setSize((int) lastSize); ++ ++ ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( ++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize); ++ ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( ++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN); ++ segment.append(buffer1, 0); ++ segment.append(buffer2, 0); ++ ++ // Mock new message arrive ++ segment.blocker = new CompletableFuture<>(); ++ new Thread(() -> { ++ try { ++ Thread.sleep(3000); ++ } catch (InterruptedException e) { ++ Assert.fail(e.getMessage()); ++ } ++ ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer(); ++ buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2); ++ buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime); ++ segment.append(buffer, 0); ++ segment.blocker.complete(false); ++ }).start(); ++ ++ for (int i = 0; i < 3; i++) { ++ segment.commit(); ++ } ++ ++ Assert.assertEquals(lastSize, segment.getCommitPosition()); ++ Assert.assertEquals(baseOffset + lastSize, segment.getCommitOffset()); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); ++ ++ segment.blocker.join(); ++ segment.blocker = null; + ++ segment.commit(); ++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitPosition()); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitOffset()); + Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); ++ ++ segment.commit(); ++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition()); + Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset()); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); + + ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN); + Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1)); +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java +index cb155cf8f..80ad41f68 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java +@@ -23,7 +23,7 @@ import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.tieredstore.common.FileSegmentType; + import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; + import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; +-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; ++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; + import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + import org.junit.Assert; + +@@ -33,6 +33,8 @@ public class MemoryFileSegment extends TieredFileSegment { + + public CompletableFuture blocker; + ++ protected int size = 0; ++ + protected boolean checkSize = true; + + public MemoryFileSegment(FileSegmentType fileType, MessageQueue messageQueue, long baseOffset, +@@ -56,6 +58,18 @@ public class MemoryFileSegment extends TieredFileSegment { + memStore.position((int) getSize()); + } + ++ public boolean isCheckSize() { ++ return checkSize; ++ } ++ ++ public void setCheckSize(boolean checkSize) { ++ this.checkSize = checkSize; ++ } ++ ++ public ByteBuffer getMemStore() { ++ return memStore; ++ } ++ + @Override + public String getPath() { + return filePath; +@@ -66,7 +80,11 @@ public class MemoryFileSegment extends TieredFileSegment { + if (checkSize) { + return 1000; + } +- return 0; ++ return size; ++ } ++ ++ public void setSize(int size) { ++ this.size = size; + } + + @Override +@@ -85,11 +103,11 @@ public class MemoryFileSegment extends TieredFileSegment { + + @Override + public CompletableFuture commit0( +- TieredFileSegmentInputStream inputStream, long position, int length, boolean append) { ++ FileSegmentInputStream inputStream, long position, int length, boolean append) { + + try { + if (blocker != null && !blocker.get()) { +- throw new IllegalStateException(); ++ throw new IllegalStateException("Commit Exception for Memory Test"); + } + } catch (InterruptedException | ExecutionException e) { + Assert.fail(e.getMessage()); +@@ -98,7 +116,6 @@ public class MemoryFileSegment extends TieredFileSegment { + Assert.assertTrue(!checkSize || position >= getSize()); + + byte[] buffer = new byte[1024]; +- + int startPos = memStore.position(); + try { + int len; +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java +index 8ac330b37..630fd2223 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java +@@ -22,7 +22,7 @@ import java.util.concurrent.ExecutionException; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.tieredstore.common.FileSegmentType; + import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; +-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; ++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; + import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + import org.junit.Assert; + +@@ -46,7 +46,7 @@ public class MemoryFileSegmentWithoutCheck extends MemoryFileSegment { + } + + @Override +- public CompletableFuture commit0(TieredFileSegmentInputStream inputStream, long position, int length, ++ public CompletableFuture commit0(FileSegmentInputStream inputStream, long position, int length, + boolean append) { + try { + if (blocker != null && !blocker.get()) { +-- +2.32.0.windows.2 + + +From d000ef947d7c99918ceba0fa451c1e29fd84ba07 Mon Sep 17 00:00:00 2001 +From: yuz10 <845238369@qq.com> +Date: Thu, 31 Aug 2023 09:41:33 +0800 +Subject: [PATCH 3/7] [ISSUE #7283] Incorrect dledger commitlog min offset + after mappedFile re delete failed (#7284) + +--- + .../apache/rocketmq/store/dledger/DLedgerCommitLog.java | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +index ec5e86d70..d5f6acdc0 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java ++++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +@@ -162,7 +162,12 @@ public class DLedgerCommitLog extends CommitLog { + if (!mappedFileQueue.getMappedFiles().isEmpty()) { + return mappedFileQueue.getMinOffset(); + } +- return dLedgerFileList.getMinOffset(); ++ for (MmapFile file : dLedgerFileList.getMappedFiles()) { ++ if (file.isAvailable()) { ++ return file.getFileFromOffset() + file.getStartPosition(); ++ } ++ } ++ return 0; + } + + @Override +-- +2.32.0.windows.2 + + +From f82718ae3b77a16b553c03f672dc971a2d5d48fa Mon Sep 17 00:00:00 2001 +From: cnScarb +Date: Thu, 31 Aug 2023 15:50:10 +0800 +Subject: [PATCH 4/7] [ISSUE #7208] fix: when deleting topic also delete its + pop retry topic (#7209) + +--- + .../processor/AdminBrokerProcessor.java | 24 ++++++++++--- + .../processor/AdminBrokerProcessorTest.java | 36 +++++++++++++++++++ + 2 files changed, 55 insertions(+), 5 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +index bbddcec2d..8fbcd3c94 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +@@ -51,6 +51,7 @@ import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; + import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; + import org.apache.rocketmq.common.AclConfig; + import org.apache.rocketmq.common.BrokerConfig; ++import org.apache.rocketmq.common.KeyBuilder; + import org.apache.rocketmq.common.LockCallback; + import org.apache.rocketmq.common.MQVersion; + import org.apache.rocketmq.common.MixAll; +@@ -542,16 +543,29 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + } + } + +- this.brokerController.getTopicConfigManager().deleteTopicConfig(requestHeader.getTopic()); +- this.brokerController.getTopicQueueMappingManager().delete(requestHeader.getTopic()); +- this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(requestHeader.getTopic()); +- this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(requestHeader.getTopic()); +- this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(requestHeader.getTopic())); ++ final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic); ++ // delete pop retry topics first ++ for (String group : groups) { ++ final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); ++ if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { ++ deleteTopicInBroker(popRetryTopic); ++ } ++ } ++ // delete topic ++ deleteTopicInBroker(topic); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + ++ private void deleteTopicInBroker(String topic) { ++ this.brokerController.getTopicConfigManager().deleteTopicConfig(topic); ++ this.brokerController.getTopicQueueMappingManager().delete(topic); ++ this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(topic); ++ this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(topic); ++ this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(topic)); ++ } ++ + private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); +diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +index d33a217f7..9d17011b6 100644 +--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +@@ -29,6 +29,7 @@ import java.util.HashMap; + import java.util.Map; + import java.util.Properties; + import java.util.Set; ++import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.atomic.LongAdder; + import org.apache.rocketmq.broker.BrokerController; +@@ -41,6 +42,7 @@ import org.apache.rocketmq.broker.topic.RocksDBTopicConfigManager; + import org.apache.rocketmq.broker.topic.TopicConfigManager; + import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.BrokerConfig; ++import org.apache.rocketmq.common.KeyBuilder; + import org.apache.rocketmq.common.MixAll; + import org.apache.rocketmq.common.TopicConfig; + import org.apache.rocketmq.common.TopicFilterType; +@@ -90,8 +92,11 @@ import static org.assertj.core.api.Assertions.assertThat; + import static org.mockito.ArgumentMatchers.any; + import static org.mockito.ArgumentMatchers.anyInt; + import static org.mockito.ArgumentMatchers.anyLong; ++import static org.mockito.ArgumentMatchers.anySet; + import static org.mockito.ArgumentMatchers.anyString; + import static org.mockito.Mockito.mock; ++import static org.mockito.Mockito.times; ++import static org.mockito.Mockito.verify; + import static org.mockito.Mockito.when; + + @RunWith(MockitoJUnitRunner.class) +@@ -321,6 +326,37 @@ public class AdminBrokerProcessorTest { + "please execute it from master broker."); + } + ++ @Test ++ public void testDeleteWithPopRetryTopic() throws Exception { ++ String topic = "topicA"; ++ String anotherTopic = "another_topicA"; ++ ++ topicConfigManager = mock(TopicConfigManager.class); ++ when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); ++ final ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); ++ topicConfigTable.put(topic, new TopicConfig()); ++ topicConfigTable.put(KeyBuilder.buildPopRetryTopic(topic, "cid1"), new TopicConfig()); ++ ++ topicConfigTable.put(anotherTopic, new TopicConfig()); ++ topicConfigTable.put(KeyBuilder.buildPopRetryTopic(anotherTopic, "cid2"), new TopicConfig()); ++ when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigTable); ++ when(topicConfigManager.selectTopicConfig(anyString())).thenAnswer(invocation -> { ++ final String selectTopic = invocation.getArgument(0); ++ return topicConfigManager.getTopicConfigTable().get(selectTopic); ++ }); ++ ++ when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); ++ when(consumerOffsetManager.whichGroupByTopic(topic)).thenReturn(Sets.newHashSet("cid1")); ++ ++ RemotingCommand request = buildDeleteTopicRequest(topic); ++ RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); ++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); ++ ++ verify(topicConfigManager).deleteTopicConfig(topic); ++ verify(topicConfigManager).deleteTopicConfig(KeyBuilder.buildPopRetryTopic(topic, "cid1")); ++ verify(messageStore, times(2)).deleteTopics(anySet()); ++ } ++ + @Test + public void testGetAllTopicConfigInRocksdb() throws Exception { + if (notToBeExecuted()) { +-- +2.32.0.windows.2 + + +From 31d10385d1616445478104ce9ef463a8c4852ba2 Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Mon, 4 Sep 2023 14:09:32 +0800 +Subject: [PATCH 5/7] [ISSUE #7289] Fixed asynchronous send backpressure + capability + +Co-authored-by: guyinyou +--- + .../impl/producer/DefaultMQProducerImpl.java | 77 +++++++++++++------ + 1 file changed, 53 insertions(+), 24 deletions(-) + +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +index bbbb17b07..2d6b83ac2 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +@@ -547,6 +547,8 @@ public class DefaultMQProducerImpl implements MQProducerInner { + @Deprecated + public void send(final Message msg, final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException { ++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); ++ + final long beginStartTime = System.currentTimeMillis(); + Runnable runnable = new Runnable() { + @Override +@@ -554,20 +556,53 @@ public class DefaultMQProducerImpl implements MQProducerInner { + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { +- sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime); ++ sendDefaultImpl(msg, CommunicationMode.ASYNC, newCallBack, timeout - costTime); + } catch (Exception e) { +- sendCallback.onException(e); ++ newCallBack.onException(e); + } + } else { +- sendCallback.onException( ++ newCallBack.onException( + new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout")); + } + } + }; +- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); ++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); + } + +- public void executeAsyncMessageSend(Runnable runnable, final Message msg, final SendCallback sendCallback, ++ class BackpressureSendCallBack implements SendCallback { ++ public boolean isSemaphoreAsyncSizeAquired = false; ++ public boolean isSemaphoreAsyncNumAquired = false; ++ public int msgLen; ++ private final SendCallback sendCallback; ++ ++ public BackpressureSendCallBack(final SendCallback sendCallback) { ++ this.sendCallback = sendCallback; ++ } ++ ++ @Override ++ public void onSuccess(SendResult sendResult) { ++ if (isSemaphoreAsyncSizeAquired) { ++ semaphoreAsyncSendSize.release(msgLen); ++ } ++ if (isSemaphoreAsyncNumAquired) { ++ semaphoreAsyncSendNum.release(); ++ } ++ sendCallback.onSuccess(sendResult); ++ } ++ ++ @Override ++ public void onException(Throwable e) { ++ if (isSemaphoreAsyncSizeAquired) { ++ semaphoreAsyncSendSize.release(msgLen); ++ } ++ if (isSemaphoreAsyncNumAquired) { ++ semaphoreAsyncSendNum.release(); ++ } ++ sendCallback.onException(e); ++ } ++ } ++ ++ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final BackpressureSendCallBack sendCallback, + final long timeout, final long beginStartTime) + throws MQClientException, InterruptedException { + ExecutorService executor = this.getAsyncSenderExecutor(); +@@ -595,7 +630,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { + return; + } + } +- ++ sendCallback.isSemaphoreAsyncSizeAquired = isSemaphoreAsyncSizeAquired; ++ sendCallback.isSemaphoreAsyncNumAquired = isSemaphoreAsyncNumAquired; ++ sendCallback.msgLen = msgLen; + executor.submit(runnable); + } catch (RejectedExecutionException e) { + if (isEnableBackpressureForAsyncMode) { +@@ -603,15 +640,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + } else { + throw new MQClientException("executor rejected ", e); + } +- } finally { +- if (isSemaphoreAsyncSizeAquired) { +- semaphoreAsyncSendSize.release(msgLen); +- } +- if (isSemaphoreAsyncNumAquired) { +- semaphoreAsyncSendNum.release(); +- } + } +- + } + + public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg, +@@ -1188,7 +1217,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + @Deprecated + public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException { +- ++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); + final long beginStartTime = System.currentTimeMillis(); + Runnable runnable = new Runnable() { + @Override +@@ -1203,22 +1232,22 @@ public class DefaultMQProducerImpl implements MQProducerInner { + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { +- sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, ++ sendKernelImpl(msg, mq, CommunicationMode.ASYNC, newCallBack, null, + timeout - costTime); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } else { +- sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); ++ newCallBack.onException(new RemotingTooMuchRequestException("call timeout")); + } + } catch (Exception e) { +- sendCallback.onException(e); ++ newCallBack.onException(e); + } + } + + }; + +- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); ++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); + } + + /** +@@ -1315,7 +1344,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + public void send(final Message msg, final MessageQueueSelector selector, final Object arg, + final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException { +- ++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); + final long beginStartTime = System.currentTimeMillis(); + Runnable runnable = new Runnable() { + @Override +@@ -1324,21 +1353,21 @@ public class DefaultMQProducerImpl implements MQProducerInner { + if (timeout > costTime) { + try { + try { +- sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, ++ sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, newCallBack, + timeout - costTime); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } catch (Exception e) { +- sendCallback.onException(e); ++ newCallBack.onException(e); + } + } else { +- sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); ++ newCallBack.onException(new RemotingTooMuchRequestException("call timeout")); + } + } + + }; +- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); ++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); + } + + /** +-- +2.32.0.windows.2 + + +From d67b9d64cbd53824798af57ba18770e0fcefa37a Mon Sep 17 00:00:00 2001 +From: yuz10 <845238369@qq.com> +Date: Wed, 6 Sep 2023 14:07:23 +0800 +Subject: [PATCH 6/7] [ISSUE #7302] Fix singleTopicRegister code deleted in + merge + +--- + .../apache/rocketmq/broker/topic/TopicConfigManager.java | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +index 1c3b9711f..4e3c1736c 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +@@ -330,7 +330,11 @@ public class TopicConfigManager extends ConfigManager { + log.error("createTopicIfAbsent ", e); + } + if (createNew && register) { +- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++ this.brokerController.registerSingleTopicAll(topicConfig); ++ } else { ++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++ } + } + return getTopicConfig(topicConfig.getTopicName()); + } +-- +2.32.0.windows.2 + + +From 37017dbaec5c521fd529ef4aecf3658092884f84 Mon Sep 17 00:00:00 2001 +From: lizhimins <707364882@qq.com> +Date: Wed, 6 Sep 2023 15:23:15 +0800 +Subject: [PATCH 7/7] [ISSUE #7305] Fix metrics and transactional module not + shutdown while broker offline cause coredump(#7307) + +--- + .../java/org/apache/rocketmq/broker/BrokerController.java | 8 ++++++++ + .../queue/TransactionalMessageServiceImpl.java | 4 +++- + 2 files changed, 11 insertions(+), 1 deletion(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index e8f943702..6aba70cb2 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -1302,6 +1302,10 @@ public class BrokerController { + this.fastRemotingServer.shutdown(); + } + ++ if (this.brokerMetricsManager != null) { ++ this.brokerMetricsManager.shutdown(); ++ } ++ + if (this.brokerStatsManager != null) { + this.brokerStatsManager.shutdown(); + } +@@ -1324,6 +1328,10 @@ public class BrokerController { + this.ackMessageProcessor.shutdownPopReviveService(); + } + ++ if (this.transactionalMessageService != null) { ++ this.transactionalMessageService.close(); ++ } ++ + if (this.notificationProcessor != null) { + this.notificationProcessor.getPopLongPollingService().shutdown(); + } +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +index 93fa725a9..48db828e0 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +@@ -629,7 +629,9 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ + + @Override + public void close() { +- ++ if (this.transactionalOpBatchService != null) { ++ this.transactionalOpBatchService.shutdown(); ++ } + } + + public Message getOpMessage(int queueId, String moreData) { +-- +2.32.0.windows.2 + diff --git a/patch016-backport-Optimize-fault-tolerant-mechanism.patch b/patch016-backport-Optimize-fault-tolerant-mechanism.patch new file mode 100644 index 000000000..9d8ef20a1 --- /dev/null +++ b/patch016-backport-Optimize-fault-tolerant-mechanism.patch @@ -0,0 +1,520 @@ +From e11e29419f6e2d1d9673d0329e57b824ebf3da47 Mon Sep 17 00:00:00 2001 +From: lizhimins <707364882@qq.com> +Date: Wed, 6 Sep 2023 20:42:24 +0800 +Subject: [PATCH 1/3] [ISSUE #7308] Adding topic blacklist and filter in tiered + storage module (#7310) + +--- + .../tieredstore/TieredDispatcher.java | 21 +++++++-- + .../tieredstore/TieredMessageStore.java | 1 + + .../file/TieredFlatFileManager.java | 17 ++++--- + .../TieredStoreTopicBlackListFilter.java | 45 +++++++++++++++++++ + .../provider/TieredStoreTopicFilter.java | 25 +++++++++++ + .../TieredStoreTopicBlackListFilterTest.java | 36 +++++++++++++++ + 6 files changed, 136 insertions(+), 9 deletions(-) + create mode 100644 tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java + create mode 100644 tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java + create mode 100644 tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +index 430c2b62e..766c559e9 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +@@ -48,6 +48,8 @@ import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile; + import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager; + import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant; + import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; ++import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicBlackListFilter; ++import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicFilter; + import org.apache.rocketmq.tieredstore.util.CQItemBufferUtil; + import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; + import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; +@@ -56,6 +58,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch + + private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); + ++ private TieredStoreTopicFilter topicFilter; + private final String brokerName; + private final MessageStore defaultStore; + private final TieredMessageStoreConfig storeConfig; +@@ -70,15 +73,15 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch + this.defaultStore = defaultStore; + this.storeConfig = storeConfig; + this.brokerName = storeConfig.getBrokerName(); ++ this.topicFilter = new TieredStoreTopicBlackListFilter(); + this.tieredFlatFileManager = TieredFlatFileManager.getInstance(storeConfig); + this.dispatchRequestReadMap = new ConcurrentHashMap<>(); + this.dispatchRequestWriteMap = new ConcurrentHashMap<>(); + this.dispatchTaskLock = new ReentrantLock(); + this.dispatchWriteLock = new ReentrantLock(); +- this.initScheduleTask(); + } + +- private void initScheduleTask() { ++ protected void initScheduleTask() { + TieredStoreExecutor.commonScheduledExecutor.scheduleWithFixedDelay(() -> + tieredFlatFileManager.deepCopyFlatFileToList().forEach(flatFile -> { + if (!flatFile.getCompositeFlatFileLock().isLocked()) { +@@ -87,6 +90,14 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch + }), 30, 10, TimeUnit.SECONDS); + } + ++ public TieredStoreTopicFilter getTopicFilter() { ++ return topicFilter; ++ } ++ ++ public void setTopicFilter(TieredStoreTopicFilter topicFilter) { ++ this.topicFilter = topicFilter; ++ } ++ + @Override + public void dispatch(DispatchRequest request) { + if (stopped) { +@@ -94,7 +105,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch + } + + String topic = request.getTopic(); +- if (TieredStoreUtil.isSystemTopic(topic)) { ++ if (topicFilter != null && topicFilter.filterTopic(topic)) { + return; + } + +@@ -219,6 +230,10 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch + return; + } + ++ if (topicFilter != null && topicFilter.filterTopic(flatFile.getMessageQueue().getTopic())) { ++ return; ++ } ++ + if (flatFile.getDispatchOffset() == -1L) { + return; + } +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +index 78e855f36..9fb1b2f01 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +@@ -90,6 +90,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + boolean loadNextStore = next.load(); + boolean result = loadFlatFile && loadNextStore; + if (result) { ++ dispatcher.initScheduleTask(); + dispatcher.start(); + } + return result; +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +index e9ae4a5a5..7c744af3b 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +@@ -134,21 +134,21 @@ public class TieredFlatFileManager { + public void doCleanExpiredFile() { + long expiredTimeStamp = System.currentTimeMillis() - + TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime()); +- Random random = new Random(); + for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) { +- int delay = random.nextInt(storeConfig.getMaxCommitJitter()); +- TieredStoreExecutor.cleanExpiredFileExecutor.schedule(() -> { ++ TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> { + flatFile.getCompositeFlatFileLock().lock(); + try { + flatFile.cleanExpiredFile(expiredTimeStamp); + flatFile.destroyExpiredFile(); + if (flatFile.getConsumeQueueBaseOffset() == -1) { ++ logger.info("Clean flatFile because file not initialized, topic={}, queueId={}", ++ flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId()); + destroyCompositeFile(flatFile.getMessageQueue()); + } + } finally { + flatFile.getCompositeFlatFileLock().unlock(); + } +- }, delay, TimeUnit.MILLISECONDS); ++ }); + } + if (indexFile != null) { + indexFile.cleanExpiredFile(expiredTimeStamp); +@@ -218,8 +218,13 @@ public class TieredFlatFileManager { + storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId())); + queueCount.incrementAndGet(); + }); +- logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", +- topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); ++ ++ if (queueCount.get() == 0L) { ++ metadataStore.deleteTopic(topicMetadata.getTopic()); ++ } else { ++ logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", ++ topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); ++ } + } catch (Exception e) { + logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e); + } finally { +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java +new file mode 100644 +index 000000000..50adbb713 +--- /dev/null ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java +@@ -0,0 +1,45 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.tieredstore.provider; ++ ++import java.util.HashSet; ++import java.util.Set; ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ ++public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter { ++ ++ private final Set topicBlackSet; ++ ++ public TieredStoreTopicBlackListFilter() { ++ this.topicBlackSet = new HashSet<>(); ++ } ++ ++ @Override ++ public boolean filterTopic(String topicName) { ++ if (StringUtils.isBlank(topicName)) { ++ return true; ++ } ++ return TieredStoreUtil.isSystemTopic(topicName) || topicBlackSet.contains(topicName); ++ } ++ ++ @Override ++ public void addTopicToWhiteList(String topicName) { ++ this.topicBlackSet.add(topicName); ++ } ++} +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java +new file mode 100644 +index 000000000..3f26b8b02 +--- /dev/null ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java +@@ -0,0 +1,25 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.tieredstore.provider; ++ ++public interface TieredStoreTopicFilter { ++ ++ boolean filterTopic(String topicName); ++ ++ void addTopicToWhiteList(String topicName); ++} +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java +new file mode 100644 +index 000000000..2bf48173c +--- /dev/null ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java +@@ -0,0 +1,36 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.tieredstore.provider; ++ ++import org.apache.rocketmq.common.topic.TopicValidator; ++import org.junit.Assert; ++import org.junit.Test; ++ ++public class TieredStoreTopicBlackListFilterTest { ++ ++ @Test ++ public void filterTopicTest() { ++ TieredStoreTopicFilter topicFilter = new TieredStoreTopicBlackListFilter(); ++ Assert.assertTrue(topicFilter.filterTopic("")); ++ Assert.assertTrue(topicFilter.filterTopic(TopicValidator.SYSTEM_TOPIC_PREFIX + "_Topic")); ++ ++ String topicName = "WhiteTopic"; ++ Assert.assertFalse(topicFilter.filterTopic(topicName)); ++ topicFilter.addTopicToWhiteList(topicName); ++ Assert.assertTrue(topicFilter.filterTopic(topicName)); ++ } ++} +\ No newline at end of file +-- +2.32.0.windows.2 + + +From 628020537fa7035226bc8dcde9fa33d9d5df30ff Mon Sep 17 00:00:00 2001 +From: rongtong +Date: Thu, 7 Sep 2023 16:17:47 +0800 +Subject: [PATCH 2/3] [ISSUE #7293] Fix NPE when alter sync state set + +--- + .../rocketmq/controller/impl/manager/ReplicasInfoManager.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java +index b0a67531d..d83a690f9 100644 +--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java ++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java +@@ -104,7 +104,7 @@ public class ReplicasInfoManager { + } + + // Check master +- if (!syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) { ++ if (syncStateInfo.getMasterBrokerId() == null || !syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) { + String err = String.format("Rejecting alter syncStateSet request because the current leader is:{%s}, not {%s}", + syncStateInfo.getMasterBrokerId(), request.getMasterBrokerId()); + LOGGER.error("{}", err); +-- +2.32.0.windows.2 + + +From 6fd0073d6475c539e8f4c30dc4f104a56a21d724 Mon Sep 17 00:00:00 2001 +From: Ji Juntao +Date: Thu, 7 Sep 2023 20:21:16 +0800 +Subject: [PATCH 3/3] [ISSUE #7319] Optimize fault-tolerant mechanism for + sending messages and hot update switch (#7320) + +--- + .../impl/producer/DefaultMQProducerImpl.java | 8 ++------ + .../client/latency/LatencyFaultTolerance.java | 14 +++++++++++++ + .../latency/LatencyFaultToleranceImpl.java | 13 +++++++++++- + .../client/latency/MQFaultStrategy.java | 20 +++++++------------ + .../proxy/service/route/MessageQueueView.java | 9 --------- + .../service/route/TopicRouteService.java | 10 +++++++++- + 6 files changed, 44 insertions(+), 30 deletions(-) + +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +index 2d6b83ac2..b0c212e46 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +@@ -263,9 +263,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + mQClientFactory.start(); + } + +- if (this.mqFaultStrategy.isStartDetectorEnable()) { +- this.mqFaultStrategy.startDetector(); +- } ++ this.mqFaultStrategy.startDetector(); + + log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), + this.defaultMQProducer.isSendMessageWithVIPChannel()); +@@ -311,9 +309,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { + if (shutdownFactory) { + this.mQClientFactory.shutdown(); + } +- if (this.mqFaultStrategy.isStartDetectorEnable()) { +- this.mqFaultStrategy.shutdown(); +- } ++ this.mqFaultStrategy.shutdown(); + RequestFutureHolder.getInstance().shutdown(this); + log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; +diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java +index 72d2f3450..17aaa266a 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java +@@ -89,4 +89,18 @@ public interface LatencyFaultTolerance { + * @param detectInterval each broker's detecting interval + */ + void setDetectInterval(final int detectInterval); ++ ++ /** ++ * Use it to set the detector work or not. ++ * ++ * @param startDetectorEnable set the detector's work status ++ */ ++ void setStartDetectorEnable(final boolean startDetectorEnable); ++ ++ /** ++ * Use it to judge if the detector enabled. ++ * ++ * @return is the detector should be started. ++ */ ++ boolean isStartDetectorEnable(); + } +diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +index 8af629574..d3ff7eb45 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +@@ -37,6 +37,8 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance + private int detectTimeout = 200; + private int detectInterval = 2000; + private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex(); ++ ++ private volatile boolean startDetectorEnable = false; + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { +@@ -80,7 +82,9 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance + @Override + public void run() { + try { +- detectByOneRound(); ++ if (startDetectorEnable) { ++ detectByOneRound(); ++ } + } catch (Exception e) { + log.warn("Unexpected exception raised while detecting service reachability", e); + } +@@ -137,6 +141,13 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance + this.faultItemTable.remove(name); + } + ++ public boolean isStartDetectorEnable() { ++ return startDetectorEnable; ++ } ++ ++ public void setStartDetectorEnable(boolean startDetectorEnable) { ++ this.startDetectorEnable = startDetectorEnable; ++ } + @Override + public String pickOneAtLeast() { + final Enumeration elements = this.faultItemTable.elements(); +diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +index c01490784..69fb533e5 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +@@ -24,8 +24,8 @@ import org.apache.rocketmq.common.message.MessageQueue; + + public class MQFaultStrategy { + private LatencyFaultTolerance latencyFaultTolerance; +- private boolean sendLatencyFaultEnable; +- private boolean startDetectorEnable; ++ private volatile boolean sendLatencyFaultEnable; ++ private volatile boolean startDetectorEnable; + private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L}; + private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L}; + +@@ -64,11 +64,11 @@ public class MQFaultStrategy { + + + public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) { +- this.setStartDetectorEnable(cc.isStartDetectorEnable()); +- this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); + this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector); + this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); + this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); ++ this.setStartDetectorEnable(cc.isStartDetectorEnable()); ++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); + } + + // For unit test. +@@ -123,21 +123,15 @@ public class MQFaultStrategy { + + public void setStartDetectorEnable(boolean startDetectorEnable) { + this.startDetectorEnable = startDetectorEnable; ++ this.latencyFaultTolerance.setStartDetectorEnable(startDetectorEnable); + } + + public void startDetector() { +- // user should start the detector +- // and the thread should not be in running state. +- if (this.sendLatencyFaultEnable && this.startDetectorEnable) { +- // start the detector. +- this.latencyFaultTolerance.startDetector(); +- } ++ this.latencyFaultTolerance.startDetector(); + } + + public void shutdown() { +- if (this.sendLatencyFaultEnable && this.startDetectorEnable) { +- this.latencyFaultTolerance.shutdown(); +- } ++ this.latencyFaultTolerance.shutdown(); + } + + public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java +index 8b3c2f7c8..898e529f8 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java +@@ -26,7 +26,6 @@ public class MessageQueueView { + private final MessageQueueSelector readSelector; + private final MessageQueueSelector writeSelector; + private final TopicRouteWrapper topicRouteWrapper; +- private MQFaultStrategy mqFaultStrategy; + + public MessageQueueView(String topic, TopicRouteData topicRouteData, MQFaultStrategy mqFaultStrategy) { + this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic); +@@ -67,12 +66,4 @@ public class MessageQueueView { + .add("topicRouteWrapper", topicRouteWrapper) + .toString(); + } +- +- public MQFaultStrategy getMQFaultStrategy() { +- return mqFaultStrategy; +- } +- +- public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { +- this.mqFaultStrategy = mqFaultStrategy; +- } + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +index 74769a423..caf62a1e0 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +@@ -127,7 +127,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { + @Override + public String resolve(String name) { + try { +- String brokerAddr = getBrokerAddr(null, name); ++ String brokerAddr = getBrokerAddr(ProxyContext.createForInner("MQFaultStrategy"), name); + return brokerAddr; + } catch (Exception e) { + return null; +@@ -175,9 +175,17 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { + + public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, + boolean reachable) { ++ checkSendFaultToleranceEnable(); + this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); + } + ++ public void checkSendFaultToleranceEnable() { ++ boolean hotLatencySwitch = ConfigurationManager.getProxyConfig().isSendLatencyEnable(); ++ boolean hotDetectorSwitch = ConfigurationManager.getProxyConfig().isStartDetectorEnable(); ++ this.mqFaultStrategy.setSendLatencyFaultEnable(hotLatencySwitch); ++ this.mqFaultStrategy.setStartDetectorEnable(hotDetectorSwitch); ++ } ++ + public MQFaultStrategy getMqFaultStrategy() { + return this.mqFaultStrategy; + } +-- +2.32.0.windows.2 + diff --git a/patch017-backport-Convergent-thread-pool-creation.patch b/patch017-backport-Convergent-thread-pool-creation.patch new file mode 100644 index 000000000..92d0bd2e9 --- /dev/null +++ b/patch017-backport-Convergent-thread-pool-creation.patch @@ -0,0 +1,2243 @@ +From c100d815d754d7cb330bc63e145bafd2d9b59cb1 Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Mon, 11 Sep 2023 10:13:56 +0800 +Subject: [PATCH 1/6] [ISSUE #7328] Convergent thread pool creation (#7329) + +* Convergence thread pool creation to facilitate subsequent iteration management + +* Convergence thread pool creation in ThreadPoolMonitor.java + +* fix unit test + +* Convergence ThreadPool constructor + +* Convergence ScheduledThreadPool constructor + +* remove unused import + +* Convergence ScheduledThreadPool constructor + +* remove unused import + +--------- +--- + .../rocketmq/broker/BrokerController.java | 39 +++++----- + .../client/ClientHousekeepingService.java | 4 +- + .../DefaultConsumerIdsChangeListener.java | 3 +- + .../broker/controller/ReplicasManager.java | 9 +-- + .../dledger/DLedgerRoleChangeHandler.java | 4 +- + .../broker/failover/EscapeBridge.java | 4 +- + .../broker/latency/BrokerFastFailure.java | 5 +- + .../BrokerFixedThreadPoolExecutor.java | 57 -------------- + .../broker/latency/FutureTaskExt.java | 39 ---------- + .../rocketmq/broker/out/BrokerOuterAPI.java | 7 +- + .../schedule/ScheduleMessageService.java | 7 +- + .../broker/topic/TopicRouteInfoManager.java | 4 +- + ...ractTransactionalMessageCheckListener.java | 4 +- + .../rocketmq/broker/BrokerControllerTest.java | 2 +- + .../broker/latency/BrokerFastFailureTest.java | 1 + + .../common/config/AbstractRocksDBStorage.java | 6 +- + .../FutureTaskExtThreadPoolExecutor.java | 3 +- + .../common/thread/ThreadPoolMonitor.java | 6 +- + .../rocketmq/common/utils/ThreadUtils.java | 74 ++++++++++++++++--- + .../rocketmq/container/BrokerContainer.java | 6 +- + .../controller/ControllerManager.java | 14 +--- + .../controller/impl/DLedgerController.java | 10 +-- + .../DefaultBrokerHeartbeatManager.java | 3 +- + .../rocketmq/namesrv/NamesrvController.java | 22 ++---- + .../grpc/v2/channel/GrpcChannelManager.java | 6 +- + .../remoting/RemotingProtocolServer.java | 4 +- + .../proxy/service/ClusterServiceManager.java | 12 +-- + .../proxy/service/LocalServiceManager.java | 4 +- + .../receipt/DefaultReceiptHandleManager.java | 8 +- + .../service/route/TopicRouteService.java | 9 +-- + .../remoting/netty/NettyRemotingClient.java | 4 +- + .../remoting/netty/NettyRemotingServer.java | 4 +- + .../rocketmq/store/DefaultMessageStore.java | 8 +- + .../ha/autoswitch/AutoSwitchHAService.java | 38 +++++----- + .../rocketmq/store/kv/CompactionStore.java | 21 +++--- + .../store/queue/ConsumeQueueStore.java | 4 +- + .../store/stats/BrokerStatsManager.java | 14 ++-- + .../store/timer/TimerMessageStore.java | 6 +- + .../apache/rocketmq/test/util/StatUtil.java | 1 - + .../common/TieredStoreExecutor.java | 14 ++-- + .../tools/admin/DefaultMQAdminExtImpl.java | 3 +- + 41 files changed, 215 insertions(+), 278 deletions(-) + delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java + delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index 6aba70cb2..275b64b1a 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -34,7 +34,6 @@ import org.apache.rocketmq.broker.failover.EscapeBridge; + import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap; + import org.apache.rocketmq.broker.filter.ConsumerFilterManager; + import org.apache.rocketmq.broker.latency.BrokerFastFailure; +-import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; + import org.apache.rocketmq.broker.longpolling.LmqPullRequestHoldService; + import org.apache.rocketmq.broker.longpolling.NotifyMessageArrivingListener; + import org.apache.rocketmq.broker.longpolling.PullRequestHoldService; +@@ -98,6 +97,7 @@ import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.message.MessageExtBrokerInner; + import org.apache.rocketmq.common.stats.MomentStatsItem; + import org.apache.rocketmq.common.utils.ServiceProvider; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.Configuration; +@@ -160,7 +160,6 @@ import java.util.concurrent.ExecutorService; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ScheduledFuture; +-import java.util.concurrent.ScheduledThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.locks.Lock; + import java.util.concurrent.locks.ReentrantLock; +@@ -455,10 +454,10 @@ public class BrokerController { + * Initialize resources including remoting server and thread executors. + */ + protected void initializeResources() { +- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, ++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, + new ThreadFactoryImpl("BrokerControllerScheduledThread", true, getBrokerIdentity())); + +- this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor( ++ this.sendMessageExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getSendMessageThreadPoolNums(), + this.brokerConfig.getSendMessageThreadPoolNums(), + 1000 * 60, +@@ -466,7 +465,7 @@ public class BrokerController { + this.sendThreadPoolQueue, + new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity())); + +- this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor( ++ this.pullMessageExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getPullMessageThreadPoolNums(), + this.brokerConfig.getPullMessageThreadPoolNums(), + 1000 * 60, +@@ -474,7 +473,7 @@ public class BrokerController { + this.pullThreadPoolQueue, + new ThreadFactoryImpl("PullMessageThread_", getBrokerIdentity())); + +- this.litePullMessageExecutor = new BrokerFixedThreadPoolExecutor( ++ this.litePullMessageExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getLitePullMessageThreadPoolNums(), + this.brokerConfig.getLitePullMessageThreadPoolNums(), + 1000 * 60, +@@ -482,7 +481,7 @@ public class BrokerController { + this.litePullThreadPoolQueue, + new ThreadFactoryImpl("LitePullMessageThread_", getBrokerIdentity())); + +- this.putMessageFutureExecutor = new BrokerFixedThreadPoolExecutor( ++ this.putMessageFutureExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getPutMessageFutureThreadPoolNums(), + this.brokerConfig.getPutMessageFutureThreadPoolNums(), + 1000 * 60, +@@ -490,7 +489,7 @@ public class BrokerController { + this.putThreadPoolQueue, + new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity())); + +- this.ackMessageExecutor = new BrokerFixedThreadPoolExecutor( ++ this.ackMessageExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getAckMessageThreadPoolNums(), + this.brokerConfig.getAckMessageThreadPoolNums(), + 1000 * 60, +@@ -498,7 +497,7 @@ public class BrokerController { + this.ackThreadPoolQueue, + new ThreadFactoryImpl("AckMessageThread_", getBrokerIdentity())); + +- this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor( ++ this.queryMessageExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getQueryMessageThreadPoolNums(), + this.brokerConfig.getQueryMessageThreadPoolNums(), + 1000 * 60, +@@ -506,7 +505,7 @@ public class BrokerController { + this.queryThreadPoolQueue, + new ThreadFactoryImpl("QueryMessageThread_", getBrokerIdentity())); + +- this.adminBrokerExecutor = new BrokerFixedThreadPoolExecutor( ++ this.adminBrokerExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getAdminBrokerThreadPoolNums(), + this.brokerConfig.getAdminBrokerThreadPoolNums(), + 1000 * 60, +@@ -514,7 +513,7 @@ public class BrokerController { + this.adminBrokerThreadPoolQueue, + new ThreadFactoryImpl("AdminBrokerThread_", getBrokerIdentity())); + +- this.clientManageExecutor = new BrokerFixedThreadPoolExecutor( ++ this.clientManageExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getClientManageThreadPoolNums(), + this.brokerConfig.getClientManageThreadPoolNums(), + 1000 * 60, +@@ -522,7 +521,7 @@ public class BrokerController { + this.clientManagerThreadPoolQueue, + new ThreadFactoryImpl("ClientManageThread_", getBrokerIdentity())); + +- this.heartbeatExecutor = new BrokerFixedThreadPoolExecutor( ++ this.heartbeatExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getHeartbeatThreadPoolNums(), + this.brokerConfig.getHeartbeatThreadPoolNums(), + 1000 * 60, +@@ -530,7 +529,7 @@ public class BrokerController { + this.heartbeatThreadPoolQueue, + new ThreadFactoryImpl("HeartbeatThread_", true, getBrokerIdentity())); + +- this.consumerManageExecutor = new BrokerFixedThreadPoolExecutor( ++ this.consumerManageExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getConsumerManageThreadPoolNums(), + this.brokerConfig.getConsumerManageThreadPoolNums(), + 1000 * 60, +@@ -538,7 +537,7 @@ public class BrokerController { + this.consumerManagerThreadPoolQueue, + new ThreadFactoryImpl("ConsumerManageThread_", true, getBrokerIdentity())); + +- this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor( ++ this.replyMessageExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getProcessReplyMessageThreadPoolNums(), + this.brokerConfig.getProcessReplyMessageThreadPoolNums(), + 1000 * 60, +@@ -546,7 +545,7 @@ public class BrokerController { + this.replyThreadPoolQueue, + new ThreadFactoryImpl("ProcessReplyMessageThread_", getBrokerIdentity())); + +- this.endTransactionExecutor = new BrokerFixedThreadPoolExecutor( ++ this.endTransactionExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getEndTransactionThreadPoolNums(), + this.brokerConfig.getEndTransactionThreadPoolNums(), + 1000 * 60, +@@ -554,7 +553,7 @@ public class BrokerController { + this.endTransactionThreadPoolQueue, + new ThreadFactoryImpl("EndTransactionThread_", getBrokerIdentity())); + +- this.loadBalanceExecutor = new BrokerFixedThreadPoolExecutor( ++ this.loadBalanceExecutor = ThreadUtils.newThreadPoolExecutor( + this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(), + this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(), + 1000 * 60, +@@ -562,9 +561,9 @@ public class BrokerController { + this.loadBalanceThreadPoolQueue, + new ThreadFactoryImpl("LoadBalanceProcessorThread_", getBrokerIdentity())); + +- this.syncBrokerMemberGroupExecutorService = new ScheduledThreadPoolExecutor(1, ++ this.syncBrokerMemberGroupExecutorService = ThreadUtils.newScheduledThreadPool(1, + new ThreadFactoryImpl("BrokerControllerSyncBrokerScheduledThread", getBrokerIdentity())); +- this.brokerHeartbeatExecutorService = new ScheduledThreadPoolExecutor(1, ++ this.brokerHeartbeatExecutorService = ThreadUtils.newScheduledThreadPool(1, + new ThreadFactoryImpl("BrokerControllerHeartbeatScheduledThread", getBrokerIdentity())); + + this.topicQueueMappingCleanService = new TopicQueueMappingCleanService(this); +@@ -828,8 +827,6 @@ public class BrokerController { + + initializeResources(); + +- registerProcessor(); +- + initializeScheduledTasks(); + + initialTransaction(); +@@ -1690,6 +1687,8 @@ public class BrokerController { + } + } + }, 10, 5, TimeUnit.SECONDS); ++ ++ registerProcessor(); + } + + protected void scheduleSendHeartbeat() { +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java +index 98e5f450f..cbb81f632 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java +@@ -18,11 +18,11 @@ package org.apache.rocketmq.broker.client; + + import io.netty.channel.Channel; + import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.broker.BrokerController; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.ChannelEventListener; +@@ -35,7 +35,7 @@ public class ClientHousekeepingService implements ChannelEventListener { + + public ClientHousekeepingService(final BrokerController brokerController) { + this.brokerController = brokerController; +- scheduledExecutorService = new ScheduledThreadPoolExecutor(1, ++ scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, + new ThreadFactoryImpl("ClientHousekeepingScheduledThread", brokerController.getBrokerIdentity())); + } + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java +index 2ce036a0f..d17a2a547 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java +@@ -22,7 +22,6 @@ import java.util.List; + import java.util.Map; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.broker.BrokerController; + import org.apache.rocketmq.common.AbstractBrokerRunnable; +@@ -37,7 +36,7 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen + private final BrokerController brokerController; + private final int cacheSize = 8096; + +- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, ++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, + ThreadUtils.newGenericThreadFactory("DefaultConsumerIdsChangeListener", true)); + + private ConcurrentHashMap> consumerChannelMap = new ConcurrentHashMap<>(cacheSize); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +index 37c82e434..a989e6e68 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +@@ -27,10 +27,8 @@ import java.util.concurrent.ArrayBlockingQueue; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.ExecutorService; +-import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ScheduledFuture; +-import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + + import org.apache.commons.lang3.StringUtils; +@@ -42,6 +40,7 @@ import org.apache.rocketmq.common.MixAll; + import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.protocol.EpochEntry; +@@ -107,9 +106,9 @@ public class ReplicasManager { + public ReplicasManager(final BrokerController brokerController) { + this.brokerController = brokerController; + this.brokerOuterAPI = brokerController.getBrokerOuterAPI(); +- this.scheduledService = Executors.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity())); +- this.executorService = Executors.newFixedThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity())); +- this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, ++ this.scheduledService = ThreadUtils.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity())); ++ this.executorService = ThreadUtils.newThreadPoolExecutor(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity())); ++ this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("ReplicasManager_scan_thread_", brokerController.getBrokerIdentity())); + this.haService = (AutoSwitchHAService) brokerController.getMessageStore().getHaService(); + this.brokerConfig = brokerController.getBrokerConfig(); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +index 75023ee1b..e6cb97640 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +@@ -21,12 +21,12 @@ import io.openmessaging.storage.dledger.DLedgerServer; + import io.openmessaging.storage.dledger.MemberState; + import io.openmessaging.storage.dledger.utils.DLedgerUtils; + import java.util.concurrent.ExecutorService; +-import java.util.concurrent.Executors; + import java.util.concurrent.Future; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.broker.BrokerController; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.store.DefaultMessageStore; +@@ -49,7 +49,7 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange + this.messageStore = messageStore; + this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog(); + this.dLegerServer = dLedgerCommitLog.getdLedgerServer(); +- this.executorService = Executors.newSingleThreadExecutor( ++ this.executorService = ThreadUtils.newSingleThreadExecutor( + new ThreadFactoryImpl("DLegerRoleChangeHandler_", brokerController.getBrokerIdentity())); + } + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java +index 7c350fc1d..6a0817480 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java +@@ -24,7 +24,6 @@ import java.util.concurrent.BlockingQueue; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.LinkedBlockingQueue; +-import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.broker.BrokerController; +@@ -43,6 +42,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.message.MessageExtBrokerInner; + import org.apache.rocketmq.common.message.MessageQueue; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.exception.RemotingException; +@@ -72,7 +72,7 @@ public class EscapeBridge { + public void start() throws Exception { + if (brokerController.getBrokerConfig().isEnableSlaveActingMaster() && brokerController.getBrokerConfig().isEnableRemoteEscape()) { + final BlockingQueue asyncSenderThreadPoolQueue = new LinkedBlockingQueue<>(50000); +- this.defaultAsyncSenderExecutor = new ThreadPoolExecutor( ++ this.defaultAsyncSenderExecutor = ThreadUtils.newThreadPoolExecutor( + Runtime.getRuntime().availableProcessors(), + Runtime.getRuntime().availableProcessors(), + 1000 * 60, +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +index d3d0bc8ba..3b6e9dc67 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +@@ -18,13 +18,14 @@ package org.apache.rocketmq.broker.latency; + + import java.util.concurrent.BlockingQueue; + import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.broker.BrokerController; + import org.apache.rocketmq.common.AbstractBrokerRunnable; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.UtilAll; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.future.FutureTaskExt; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.netty.RequestTask; +@@ -43,7 +44,7 @@ public class BrokerFastFailure { + + public BrokerFastFailure(final BrokerController brokerController) { + this.brokerController = brokerController; +- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, ++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, + new ThreadFactoryImpl("BrokerFastFailureScheduledThread", true, + brokerController == null ? null : brokerController.getBrokerConfig())); + } +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java +deleted file mode 100644 +index d2d1143a3..000000000 +--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java ++++ /dev/null +@@ -1,57 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +- +-package org.apache.rocketmq.broker.latency; +- +-import java.util.concurrent.BlockingQueue; +-import java.util.concurrent.RejectedExecutionHandler; +-import java.util.concurrent.RunnableFuture; +-import java.util.concurrent.ThreadFactory; +-import java.util.concurrent.ThreadPoolExecutor; +-import java.util.concurrent.TimeUnit; +- +-public class BrokerFixedThreadPoolExecutor extends ThreadPoolExecutor { +- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, +- final TimeUnit unit, +- final BlockingQueue workQueue) { +- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); +- } +- +- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, +- final TimeUnit unit, +- final BlockingQueue workQueue, final ThreadFactory threadFactory) { +- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); +- } +- +- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, +- final TimeUnit unit, +- final BlockingQueue workQueue, final RejectedExecutionHandler handler) { +- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler); +- } +- +- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, +- final TimeUnit unit, +- final BlockingQueue workQueue, final ThreadFactory threadFactory, +- final RejectedExecutionHandler handler) { +- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); +- } +- +- @Override +- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { +- return new FutureTaskExt<>(runnable, value); +- } +-} +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java +deleted file mode 100644 +index f132efaeb..000000000 +--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java ++++ /dev/null +@@ -1,39 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +- +-package org.apache.rocketmq.broker.latency; +- +-import java.util.concurrent.Callable; +-import java.util.concurrent.FutureTask; +- +-public class FutureTaskExt extends FutureTask { +- private final Runnable runnable; +- +- public FutureTaskExt(final Callable callable) { +- super(callable); +- this.runnable = null; +- } +- +- public FutureTaskExt(final Runnable runnable, final V result) { +- super(runnable, result); +- this.runnable = runnable; +- } +- +- public Runnable getRunnable() { +- return runnable; +- } +-} +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +index ae81e8b11..9dfb8127d 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +@@ -27,9 +27,9 @@ import java.util.concurrent.ArrayBlockingQueue; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.CopyOnWriteArrayList; + import java.util.concurrent.CountDownLatch; ++import java.util.concurrent.ExecutorService; + import java.util.concurrent.TimeUnit; + import org.apache.commons.lang3.StringUtils; +-import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; + import org.apache.rocketmq.client.consumer.PullResult; + import org.apache.rocketmq.client.consumer.PullStatus; + import org.apache.rocketmq.client.exception.MQBrokerException; +@@ -59,6 +59,7 @@ import org.apache.rocketmq.common.namesrv.DefaultTopAddressing; + import org.apache.rocketmq.common.namesrv.TopAddressing; + import org.apache.rocketmq.common.sysflag.PullSysFlag; + import org.apache.rocketmq.common.topic.TopicValidator; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.InvokeCallback; +@@ -144,7 +145,7 @@ public class BrokerOuterAPI { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final RemotingClient remotingClient; + private final TopAddressing topAddressing = new DefaultTopAddressing(MixAll.getWSAddr()); +- private final BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, ++ private final ExecutorService brokerOuterExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, + new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true)); + private final ClientMetadata clientMetadata; + private final RpcClient rpcClient; +@@ -1092,7 +1093,7 @@ public class BrokerOuterAPI { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + +- public BrokerFixedThreadPoolExecutor getBrokerOuterExecutor() { ++ public ExecutorService getBrokerOuterExecutor() { + return brokerOuterExecutor; + } + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +index 297b14207..0c2e6507b 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +@@ -26,7 +26,6 @@ import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.ExecutionException; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicBoolean; + import java.util.concurrent.atomic.AtomicInteger; +@@ -91,7 +90,7 @@ public class ScheduleMessageService extends ConfigManager { + public ScheduleMessageService(final BrokerController brokerController) { + this.brokerController = brokerController; + this.enableAsyncDeliver = brokerController.getMessageStoreConfig().isEnableScheduleAsyncDeliver(); +- scheduledPersistService = new ScheduledThreadPoolExecutor(1, ++ scheduledPersistService = ThreadUtils.newScheduledThreadPool(1, + new ThreadFactoryImpl("ScheduleMessageServicePersistThread", true, brokerController.getBrokerConfig())); + } + +@@ -134,9 +133,9 @@ public class ScheduleMessageService extends ConfigManager { + public void start() { + if (started.compareAndSet(false, true)) { + this.load(); +- this.deliverExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_")); ++ this.deliverExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_")); + if (this.enableAsyncDeliver) { +- this.handleExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_")); ++ this.handleExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_")); + } + for (Map.Entry entry : this.delayLevelTable.entrySet()) { + Integer level = entry.getKey(); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java +index b35564725..11bde5f5f 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java +@@ -23,7 +23,6 @@ import java.util.Objects; + import java.util.Set; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; +-import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.locks.Lock; +@@ -36,6 +35,7 @@ import org.apache.rocketmq.common.MixAll; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.message.MessageQueue; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.exception.RemotingException; +@@ -66,7 +66,7 @@ public class TopicRouteInfoManager { + } + + public void start() { +- this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread")); ++ this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread")); + + this.scheduledExecutorService.scheduleAtFixedRate(() -> { + try { +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java +index 771d84300..982355d78 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java +@@ -19,7 +19,6 @@ package org.apache.rocketmq.broker.transaction; + import io.netty.channel.Channel; + import java.util.concurrent.ArrayBlockingQueue; + import java.util.concurrent.ExecutorService; +-import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.broker.BrokerController; +@@ -27,6 +26,7 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.message.MessageConst; + import org.apache.rocketmq.common.message.MessageExt; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; +@@ -97,7 +97,7 @@ public abstract class AbstractTransactionalMessageCheckListener { + + public synchronized void initExecutorService() { + if (executorService == null) { +- executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), ++ executorService = ThreadUtils.newThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), + new ThreadFactoryImpl("Transaction-msg-check-thread", brokerController.getBrokerIdentity()), new CallerRunsPolicy()); + } + } +diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java +index 75ad961ce..6035a20ac 100644 +--- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java ++++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java +@@ -23,9 +23,9 @@ import java.util.concurrent.BlockingQueue; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.TimeUnit; + +-import org.apache.rocketmq.broker.latency.FutureTaskExt; + import org.apache.rocketmq.common.BrokerConfig; + import org.apache.rocketmq.common.UtilAll; ++import org.apache.rocketmq.common.future.FutureTaskExt; + import org.apache.rocketmq.remoting.netty.NettyClientConfig; + import org.apache.rocketmq.remoting.netty.NettyServerConfig; + import org.apache.rocketmq.remoting.netty.RequestTask; +diff --git a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java +index 5d0f7f9d7..31b547cf1 100644 +--- a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java ++++ b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java +@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.latency; + import java.util.concurrent.BlockingQueue; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.TimeUnit; ++import org.apache.rocketmq.common.future.FutureTaskExt; + import org.apache.rocketmq.remoting.netty.RequestTask; + import org.junit.Test; + +diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +index a720a5be3..6f19a9815 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +@@ -23,7 +23,6 @@ import java.util.List; + import java.util.Map; + import java.util.concurrent.ArrayBlockingQueue; + import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledThreadPoolExecutor; + import java.util.concurrent.Semaphore; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; +@@ -33,6 +32,7 @@ import com.google.common.collect.Maps; + + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.rocksdb.ColumnFamilyDescriptor; +@@ -82,8 +82,8 @@ public abstract class AbstractRocksDBStorage { + private volatile boolean closed; + + private final Semaphore reloadPermit = new Semaphore(1); +- private final ScheduledExecutorService reloadScheduler = new ScheduledThreadPoolExecutor(1, new ThreadFactoryImpl("RocksDBStorageReloadService_")); +- private final ThreadPoolExecutor manualCompactionThread = new ThreadPoolExecutor( ++ private final ScheduledExecutorService reloadScheduler = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("RocksDBStorageReloadService_")); ++ private final ThreadPoolExecutor manualCompactionThread = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor( + 1, 1, 1000 * 60, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue(1), + new ThreadFactoryImpl("RocksDBManualCompactionService_"), +diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java +index 411da9221..7b68873a9 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java ++++ b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java +@@ -29,7 +29,8 @@ public class FutureTaskExtThreadPoolExecutor extends ThreadPoolExecutor { + + public FutureTaskExtThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, + TimeUnit unit, +- BlockingQueue workQueue, ThreadFactory threadFactory, ++ BlockingQueue workQueue, ++ ThreadFactory threadFactory, + RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); + } +diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java +index 49d97a5d7..1bfabbffe 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java ++++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java +@@ -22,12 +22,12 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; + import java.util.Collections; + import java.util.List; + import java.util.concurrent.CopyOnWriteArrayList; +-import java.util.concurrent.Executors; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.common.UtilAll; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + +@@ -36,7 +36,7 @@ public class ThreadPoolMonitor { + private static Logger waterMarkLogger = LoggerFactory.getLogger(ThreadPoolMonitor.class); + + private static final List MONITOR_EXECUTOR = new CopyOnWriteArrayList<>(); +- private static final ScheduledExecutorService MONITOR_SCHEDULED = Executors.newSingleThreadScheduledExecutor( ++ private static final ScheduledExecutorService MONITOR_SCHEDULED = ThreadUtils.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("ThreadPoolMonitor-%d").build() + ); + +@@ -81,7 +81,7 @@ public class ThreadPoolMonitor { + String name, + int queueCapacity, + List threadPoolStatusMonitors) { +- ThreadPoolExecutor executor = new FutureTaskExtThreadPoolExecutor( ++ ThreadPoolExecutor executor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor( + corePoolSize, + maximumPoolSize, + keepAliveTime, +diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java +index 4b366d4e3..1644c6360 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java ++++ b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java +@@ -20,38 +20,94 @@ package org.apache.rocketmq.common.utils; + import java.util.concurrent.BlockingQueue; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; ++import java.util.concurrent.LinkedBlockingQueue; ++import java.util.concurrent.RejectedExecutionHandler; + import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.ScheduledThreadPoolExecutor; + import java.util.concurrent.ThreadFactory; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.thread.FutureTaskExtThreadPoolExecutor; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + + public final class ThreadUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME); + +- public static ExecutorService newThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, +- TimeUnit unit, BlockingQueue workQueue, String processName, boolean isDaemon) { +- return new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon)); ++ public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) { ++ return ThreadUtils.newSingleThreadExecutor(newThreadFactory(processName, isDaemon)); + } + +- public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) { +- return Executors.newSingleThreadExecutor(newThreadFactory(processName, isDaemon)); ++ public static ExecutorService newSingleThreadExecutor(ThreadFactory threadFactory) { ++ return ThreadUtils.newThreadPoolExecutor(1, threadFactory); ++ } ++ ++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory) { ++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, corePoolSize, ++ 0L, TimeUnit.MILLISECONDS, ++ new LinkedBlockingQueue<>(), ++ threadFactory); ++ } ++ ++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, ++ int maximumPoolSize, ++ long keepAliveTime, ++ TimeUnit unit, BlockingQueue workQueue, ++ String processName, ++ boolean isDaemon) { ++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon)); ++ } ++ ++ public static ExecutorService newThreadPoolExecutor(final int corePoolSize, ++ final int maximumPoolSize, ++ final long keepAliveTime, ++ final TimeUnit unit, ++ final BlockingQueue workQueue, ++ final ThreadFactory threadFactory) { ++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, new ThreadPoolExecutor.AbortPolicy()); ++ } ++ ++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, ++ int maximumPoolSize, ++ long keepAliveTime, ++ TimeUnit unit, ++ BlockingQueue workQueue, ++ ThreadFactory threadFactory, ++ RejectedExecutionHandler handler) { ++ return new FutureTaskExtThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); + } + + public static ScheduledExecutorService newSingleThreadScheduledExecutor(String processName, boolean isDaemon) { +- return Executors.newSingleThreadScheduledExecutor(newThreadFactory(processName, isDaemon)); ++ return ThreadUtils.newScheduledThreadPool(1, processName, isDaemon); ++ } ++ ++ public static ScheduledExecutorService newSingleThreadScheduledExecutor(ThreadFactory threadFactory) { ++ return ThreadUtils.newScheduledThreadPool(1, threadFactory); ++ } ++ ++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize) { ++ return ThreadUtils.newScheduledThreadPool(corePoolSize, Executors.defaultThreadFactory()); + } + +- public static ScheduledExecutorService newFixedThreadScheduledPool(int nThreads, String processName, ++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, String processName, + boolean isDaemon) { +- return Executors.newScheduledThreadPool(nThreads, newThreadFactory(processName, isDaemon)); ++ return ThreadUtils.newScheduledThreadPool(corePoolSize, newThreadFactory(processName, isDaemon)); ++ } ++ ++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, ThreadFactory threadFactory) { ++ return ThreadUtils.newScheduledThreadPool(corePoolSize, threadFactory, new ThreadPoolExecutor.AbortPolicy()); ++ } ++ ++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, ++ ThreadFactory threadFactory, ++ RejectedExecutionHandler handler) { ++ return new ScheduledThreadPoolExecutor(corePoolSize, threadFactory, handler); + } + + public static ThreadFactory newThreadFactory(String processName, boolean isDaemon) { +- return newGenericThreadFactory("Remoting-" + processName, isDaemon); ++ return newGenericThreadFactory("ThreadUtils-" + processName, isDaemon); + } + + public static ThreadFactory newGenericThreadFactory(String processName) { +diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java +index c6446f058..5b712bc30 100644 +--- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java ++++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java +@@ -47,14 +47,12 @@ import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledThreadPoolExecutor; +-import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + + public class BrokerContainer implements IBrokerContainer { + private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + +- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, ++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, + new BasicThreadFactory.Builder() + .namingPattern("BrokerContainerScheduledThread") + .daemon(true) +@@ -143,7 +141,7 @@ public class BrokerContainer implements IBrokerContainer { + this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.containerClientHouseKeepingService); + this.fastRemotingServer = this.remotingServer.newRemotingServer(this.nettyServerConfig.getListenPort() - 2); + +- this.brokerContainerExecutor = new ThreadPoolExecutor( ++ this.brokerContainerExecutor = ThreadUtils.newThreadPoolExecutor( + 1, + 1, + 1000 * 60, +diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java +index 7c91e70da..3e6b0eba5 100644 +--- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java ++++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java +@@ -25,8 +25,6 @@ import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; + import java.util.concurrent.Future; + import java.util.concurrent.LinkedBlockingQueue; +-import java.util.concurrent.RunnableFuture; +-import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + + import org.apache.commons.lang3.StringUtils; +@@ -34,8 +32,8 @@ import org.apache.rocketmq.common.ControllerConfig; + import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; +-import org.apache.rocketmq.common.future.FutureTaskExt; + ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy; + import org.apache.rocketmq.controller.impl.DLedgerController; + import org.apache.rocketmq.controller.impl.heartbeat.DefaultBrokerHeartbeatManager; +@@ -93,18 +91,14 @@ public class ControllerManager { + + public boolean initialize() { + this.controllerRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.controllerConfig.getControllerRequestThreadPoolQueueCapacity()); +- this.controllerRequestExecutor = new ThreadPoolExecutor( ++ this.controllerRequestExecutor = ThreadUtils.newThreadPoolExecutor( + this.controllerConfig.getControllerThreadPoolNums(), + this.controllerConfig.getControllerThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.controllerRequestThreadPoolQueue, +- new ThreadFactoryImpl("ControllerRequestExecutorThread_")) { +- @Override +- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { +- return new FutureTaskExt(runnable, value); +- } +- }; ++ new ThreadFactoryImpl("ControllerRequestExecutorThread_")); ++ + this.notifyService.initialize(); + if (StringUtils.isEmpty(this.controllerConfig.getControllerDLegerPeers())) { + throw new IllegalArgumentException("Attribute value controllerDLegerPeers of ControllerConfig is null or empty"); +diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java +index fa91f288e..33e4406e4 100644 +--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java ++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java +@@ -32,7 +32,6 @@ import java.util.Map; + import java.util.concurrent.BlockingQueue; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ExecutorService; +-import java.util.concurrent.Executors; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ScheduledFuture; +@@ -44,6 +43,7 @@ import org.apache.rocketmq.common.ControllerConfig; + import org.apache.rocketmq.common.ServiceThread; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.controller.Controller; + import org.apache.rocketmq.controller.elect.ElectPolicy; + import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy; +@@ -66,11 +66,11 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; + import org.apache.rocketmq.remoting.protocol.ResponseCode; + import org.apache.rocketmq.remoting.protocol.body.SyncStateSet; + import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader; +-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; +-import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; ++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader; ++import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader; +@@ -136,7 +136,7 @@ public class DLedgerController implements Controller { + this.dLedgerServer = new DLedgerServer(dLedgerConfig, nettyServerConfig, nettyClientConfig, channelEventListener); + this.dLedgerServer.registerStateMachine(this.statemachine); + this.dLedgerServer.getDLedgerLeaderElector().addRoleChangeHandler(this.roleHandler); +- this.scanInactiveMasterService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_")); ++ this.scanInactiveMasterService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_")); + this.brokerLifecycleListeners = new ArrayList<>(); + } + +@@ -513,7 +513,7 @@ public class DLedgerController implements Controller { + class RoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler { + + private final String selfId; +- private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_")); ++ private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_")); + private volatile MemberState.Role currentRole = MemberState.Role.FOLLOWER; + + public RoleChangeHandler(final String selfId) { +diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java +index 2fbddb9cd..6ebb2c994 100644 +--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java ++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java +@@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.common.ControllerConfig; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.controller.BrokerHeartbeatManager; + import org.apache.rocketmq.controller.helper.BrokerLifecycleListener; + import org.apache.rocketmq.logging.org.slf4j.Logger; +@@ -66,7 +67,7 @@ public class DefaultBrokerHeartbeatManager implements BrokerHeartbeatManager { + + @Override + public void initialize() { +- this.scheduledService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_")); ++ this.scheduledService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_")); + this.executor = Executors.newFixedThreadPool(2, new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_executorService_")); + } + +diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java +index 15c65ebec..be327cffa 100644 +--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java ++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java +@@ -20,10 +20,7 @@ import java.util.Collections; + import java.util.concurrent.BlockingQueue; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.LinkedBlockingQueue; +-import java.util.concurrent.RunnableFuture; + import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledThreadPoolExecutor; +-import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.commons.lang3.concurrent.BasicThreadFactory; + import org.apache.rocketmq.common.ThreadFactoryImpl; +@@ -31,6 +28,7 @@ import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.future.FutureTaskExt; + import org.apache.rocketmq.common.namesrv.NamesrvConfig; + import org.apache.rocketmq.common.utils.NetworkUtil; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.namesrv.kvconfig.KVConfigManager; +@@ -62,10 +60,10 @@ public class NamesrvController { + private final NettyServerConfig nettyServerConfig; + private final NettyClientConfig nettyClientConfig; + +- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, ++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, + new BasicThreadFactory.Builder().namingPattern("NSScheduledThread").daemon(true).build()); + +- private final ScheduledExecutorService scanExecutorService = new ScheduledThreadPoolExecutor(1, ++ private final ScheduledExecutorService scanExecutorService = ThreadUtils.newScheduledThreadPool(1, + new BasicThreadFactory.Builder().namingPattern("NSScanScheduledThread").daemon(true).build()); + + private final KVConfigManager kvConfigManager; +@@ -138,20 +136,10 @@ public class NamesrvController { + + private void initiateThreadExecutors() { + this.defaultThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getDefaultThreadPoolQueueCapacity()); +- this.defaultExecutor = new ThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_")) { +- @Override +- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { +- return new FutureTaskExt<>(runnable, value); +- } +- }; ++ this.defaultExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_")); + + this.clientRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getClientRequestThreadPoolQueueCapacity()); +- this.clientRequestExecutor = new ThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_")) { +- @Override +- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { +- return new FutureTaskExt<>(runnable, value); +- } +- }; ++ this.clientRequestExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_")); + } + + private void initiateSslContext() { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java +index 14330dd8d..a18cf7600 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java +@@ -21,13 +21,13 @@ import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; +-import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicLong; + import org.apache.rocketmq.common.ThreadFactoryImpl; +-import org.apache.rocketmq.proxy.common.ProxyContext; + import org.apache.rocketmq.common.utils.StartAndShutdown; ++import org.apache.rocketmq.common.utils.ThreadUtils; ++import org.apache.rocketmq.proxy.common.ProxyContext; + import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.config.ProxyConfig; + import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +@@ -43,7 +43,7 @@ public class GrpcChannelManager implements StartAndShutdown { + protected final AtomicLong nonceIdGenerator = new AtomicLong(0); + protected final ConcurrentMap resultNonceFutureMap = new ConcurrentHashMap<>(); + +- protected final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( ++ protected final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("GrpcChannelManager_") + ); + +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +index bcc9edd09..fe07090d5 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +@@ -22,7 +22,6 @@ import io.netty.channel.Channel; + import java.util.List; + import java.util.concurrent.BlockingQueue; + import java.util.concurrent.CompletableFuture; +-import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; +@@ -33,6 +32,7 @@ import org.apache.rocketmq.common.future.FutureTaskExt; + import org.apache.rocketmq.common.thread.ThreadPoolMonitor; + import org.apache.rocketmq.common.thread.ThreadPoolStatusMonitor; + import org.apache.rocketmq.common.utils.StartAndShutdown; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.proxy.config.ConfigurationManager; +@@ -178,7 +178,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu + new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInDefaultQueue()) + ); + +- this.timerExecutor = Executors.newSingleThreadScheduledExecutor( ++ this.timerExecutor = ThreadUtils.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("RemotingServerScheduler-%d").build() + ); + this.timerExecutor.scheduleAtFixedRate(this::cleanExpireRequest, 10, 10, TimeUnit.SECONDS); +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java +index d2ddfc352..9786cec55 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java +@@ -16,7 +16,6 @@ + */ + package org.apache.rocketmq.proxy.service; + +-import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.broker.client.ClientChannelInfo; +@@ -27,23 +26,24 @@ import org.apache.rocketmq.broker.client.ProducerChangeListener; + import org.apache.rocketmq.broker.client.ProducerGroupEvent; + import org.apache.rocketmq.broker.client.ProducerManager; + import org.apache.rocketmq.client.common.NameserverAccessConfig; ++import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor; ++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +-import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; + import org.apache.rocketmq.proxy.common.ProxyContext; + import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.config.ProxyConfig; + import org.apache.rocketmq.proxy.service.admin.AdminService; + import org.apache.rocketmq.proxy.service.admin.DefaultAdminService; + import org.apache.rocketmq.proxy.service.client.ClusterConsumerManager; ++import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor; + import org.apache.rocketmq.proxy.service.message.ClusterMessageService; + import org.apache.rocketmq.proxy.service.message.MessageService; + import org.apache.rocketmq.proxy.service.metadata.ClusterMetadataService; + import org.apache.rocketmq.proxy.service.metadata.MetadataService; +-import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor; +-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; +-import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor; + import org.apache.rocketmq.proxy.service.relay.ClusterProxyRelayService; + import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; + import org.apache.rocketmq.proxy.service.route.ClusterTopicRouteService; +@@ -73,7 +73,7 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + NameserverAccessConfig nameserverAccessConfig = new NameserverAccessConfig(proxyConfig.getNamesrvAddr(), + proxyConfig.getNamesrvDomain(), proxyConfig.getNamesrvDomainSubgroup()); +- this.scheduledExecutorService = Executors.newScheduledThreadPool(3); ++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(3); + + this.messagingClientAPIFactory = new MQClientAPIFactory( + nameserverAccessConfig, +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java +index 4d1ca7b66..59cd92685 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java +@@ -16,7 +16,6 @@ + */ + package org.apache.rocketmq.proxy.service; + +-import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.broker.BrokerController; +@@ -28,6 +27,7 @@ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; + import org.apache.rocketmq.common.utils.StartAndShutdown; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.config.ProxyConfig; + import org.apache.rocketmq.proxy.service.admin.AdminService; +@@ -58,7 +58,7 @@ public class LocalServiceManager extends AbstractStartAndShutdown implements Ser + private final MQClientAPIFactory mqClientAPIFactory; + private final ChannelManager channelManager; + +- private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( ++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("LocalServiceManagerScheduledThread")); + + public LocalServiceManager(BrokerController brokerController, RPCHook rpcHook) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java +index 69f44344a..207603fe8 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java +@@ -24,7 +24,6 @@ import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; +-import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; +@@ -42,20 +41,21 @@ import org.apache.rocketmq.common.thread.ThreadPoolMonitor; + import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; + import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; + import org.apache.rocketmq.common.utils.StartAndShutdown; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +-import org.apache.rocketmq.proxy.common.RenewEvent; + import org.apache.rocketmq.proxy.common.MessageReceiptHandle; + import org.apache.rocketmq.proxy.common.ProxyContext; + import org.apache.rocketmq.proxy.common.ProxyException; + import org.apache.rocketmq.proxy.common.ProxyExceptionCode; + import org.apache.rocketmq.proxy.common.ReceiptHandleGroup; ++import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey; ++import org.apache.rocketmq.proxy.common.RenewEvent; + import org.apache.rocketmq.proxy.common.RenewStrategyPolicy; + import org.apache.rocketmq.proxy.common.channel.ChannelHelper; + import org.apache.rocketmq.proxy.common.utils.ExceptionUtils; + import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.config.ProxyConfig; +-import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey; + import org.apache.rocketmq.proxy.service.metadata.MetadataService; + import org.apache.rocketmq.remoting.protocol.subscription.RetryPolicy; + import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +@@ -68,7 +68,7 @@ public class DefaultReceiptHandleManager extends AbstractStartAndShutdown implem + protected final StateEventListener eventListener; + protected final static RetryPolicy RENEW_POLICY = new RenewStrategyPolicy(); + protected final ScheduledExecutorService scheduledExecutorService = +- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_")); ++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_")); + protected final ThreadPoolExecutor renewalWorkerService; + + public DefaultReceiptHandleManager(MetadataService metadataService, ConsumerManager consumerManager, StateEventListener eventListener) { +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +index caf62a1e0..ccf094c03 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +@@ -19,25 +19,24 @@ package org.apache.rocketmq.proxy.service.route; + import com.github.benmanes.caffeine.cache.CacheLoader; + import com.github.benmanes.caffeine.cache.Caffeine; + import com.github.benmanes.caffeine.cache.LoadingCache; ++import com.google.common.base.Optional; + import java.time.Duration; + import java.util.List; +-import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; +- +-import com.google.common.base.Optional; + import org.apache.rocketmq.client.ClientConfig; + import org.apache.rocketmq.client.exception.MQClientException; ++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; + import org.apache.rocketmq.client.latency.MQFaultStrategy; + import org.apache.rocketmq.client.latency.Resolver; + import org.apache.rocketmq.client.latency.ServiceDetector; +-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.common.thread.ThreadPoolMonitor; + import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.proxy.common.Address; +@@ -63,7 +62,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { + public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) { + ProxyConfig config = ConfigurationManager.getProxyConfig(); + +- this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( ++ this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("TopicRouteService_") + ); + this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor( +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +index 8491f4354..64621dd6c 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +@@ -61,7 +61,6 @@ import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; +-import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicInteger; + import java.util.concurrent.atomic.AtomicReference; +@@ -71,6 +70,7 @@ import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.ChannelEventListener; +@@ -142,7 +142,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + + this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactoryImpl("NettyClientPublicExecutor_")); + +- this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, ++ this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("NettyClientScan_thread_")); + + if (eventLoopGroup != null) { +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +index e626260c9..aa0d46542 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +@@ -61,6 +61,7 @@ import org.apache.rocketmq.common.constant.HAProxyConstants; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.utils.BinaryUtil; + import org.apache.rocketmq.common.utils.NetworkUtil; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.ChannelEventListener; +@@ -83,7 +84,6 @@ import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledThreadPoolExecutor; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + +@@ -171,7 +171,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti + } + + private ScheduledExecutorService buildScheduleExecutor() { +- return new ScheduledThreadPoolExecutor(1, ++ return ThreadUtils.newScheduledThreadPool(1, + new ThreadFactoryImpl("NettyServerScheduler_", true), + new ThreadPoolExecutor.DiscardOldestPolicy()); + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +index f2a54ddf6..02ea47f13 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +@@ -48,7 +48,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; + import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.ExecutionException; + import java.util.concurrent.ExecutorService; +-import java.util.concurrent.Executors; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ThreadPoolExecutor; +@@ -83,6 +82,7 @@ import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.common.utils.CleanupPolicyUtils; + import org.apache.rocketmq.common.utils.QueueTypeUtils; + import org.apache.rocketmq.common.utils.ServiceProvider; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; +@@ -205,7 +205,7 @@ public class DefaultMessageStore implements MessageStore { + private ConcurrentMap topicConfigTable; + + private final ScheduledExecutorService scheduledCleanQueueExecutorService = +- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); ++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); + + public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, + final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws IOException { +@@ -253,7 +253,7 @@ public class DefaultMessageStore implements MessageStore { + this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog()); + + this.scheduledExecutorService = +- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); ++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); + + this.dispatcherList = new LinkedList<>(); + this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue()); +@@ -2915,7 +2915,7 @@ public class DefaultMessageStore implements MessageStore { + private final ExecutorService batchDispatchRequestExecutor; + + public MainBatchDispatchRequestService() { +- batchDispatchRequestExecutor = new ThreadPoolExecutor( ++ batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor( + DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), + DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), + 1000 * 60, +diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +index d5393fdca..f20bc3e28 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +@@ -17,10 +17,26 @@ + + package org.apache.rocketmq.store.ha.autoswitch; + +- ++import java.io.IOException; ++import java.nio.channels.SocketChannel; ++import java.util.ArrayList; ++import java.util.HashSet; ++import java.util.Iterator; ++import java.util.List; ++import java.util.Map; ++import java.util.Objects; ++import java.util.Set; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ExecutorService; ++import java.util.concurrent.locks.Lock; ++import java.util.concurrent.locks.ReadWriteLock; ++import java.util.concurrent.locks.ReentrantReadWriteLock; ++import java.util.function.Consumer; ++import java.util.stream.Collectors; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.protocol.EpochEntry; +@@ -36,30 +52,12 @@ import org.apache.rocketmq.store.ha.HAClient; + import org.apache.rocketmq.store.ha.HAConnection; + import org.apache.rocketmq.store.ha.HAConnectionStateNotificationService; + +-import java.io.IOException; +-import java.nio.channels.SocketChannel; +-import java.util.ArrayList; +-import java.util.HashSet; +-import java.util.List; +-import java.util.Iterator; +-import java.util.Map; +-import java.util.Objects; +-import java.util.Set; +-import java.util.concurrent.ConcurrentHashMap; +-import java.util.concurrent.ExecutorService; +-import java.util.concurrent.Executors; +-import java.util.concurrent.locks.Lock; +-import java.util.concurrent.locks.ReadWriteLock; +-import java.util.concurrent.locks.ReentrantReadWriteLock; +-import java.util.function.Consumer; +-import java.util.stream.Collectors; +- + /** + * SwitchAble ha service, support switch role to master or slave. + */ + public class AutoSwitchHAService extends DefaultHAService { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); +- private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_")); ++ private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_")); + private final ConcurrentHashMap connectionCaughtUpTimeTable = new ConcurrentHashMap<>(); + private final List>> syncStateSetChangedListeners = new ArrayList<>(); + private final Set syncStateSet = new HashSet<>(); +diff --git a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java +index b37c90726..639084fa2 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java +@@ -16,17 +16,25 @@ + */ + package org.apache.rocketmq.store.kv; + +-import java.util.Random; ++import java.io.File; ++import java.io.IOException; ++import java.nio.file.Files; ++import java.nio.file.Paths; + import java.util.Iterator; + import java.util.List; + import java.util.Map; + import java.util.Objects; + import java.util.Optional; ++import java.util.Random; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.TopicConfig; + import org.apache.rocketmq.common.attribute.CleanupPolicy; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.utils.CleanupPolicyUtils; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.store.DefaultMessageStore; +@@ -35,15 +43,6 @@ import org.apache.rocketmq.store.GetMessageResult; + import org.apache.rocketmq.store.SelectMappedBufferResult; + import org.apache.rocketmq.store.config.MessageStoreConfig; + +-import java.io.File; +-import java.io.IOException; +-import java.nio.file.Files; +-import java.nio.file.Paths; +-import java.util.concurrent.ConcurrentHashMap; +-import java.util.concurrent.Executors; +-import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.TimeUnit; +- + public class CompactionStore { + + public static final String COMPACTION_DIR = "compaction"; +@@ -76,7 +75,7 @@ public class CompactionStore { + this.positionMgr = new CompactionPositionMgr(compactionPath); + this.compactionThreadNum = Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, config.getCompactionThreadNum())); + +- this.compactionSchedule = Executors.newScheduledThreadPool(this.compactionThreadNum, ++ this.compactionSchedule = ThreadUtils.newScheduledThreadPool(this.compactionThreadNum, + new ThreadFactoryImpl("compactionSchedule_")); + this.offsetMapSize = config.getMaxOffsetMapSize() / compactionThreadNum; + +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +index 8d38503b3..d03d15d65 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +@@ -24,7 +24,6 @@ import java.util.concurrent.CountDownLatch; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.FutureTask; + import java.util.concurrent.LinkedBlockingQueue; +-import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.TopicConfig; +@@ -34,6 +33,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.common.utils.QueueTypeUtils; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.store.CommitLog; +@@ -175,7 +175,7 @@ public class ConsumeQueueStore { + } + + private ExecutorService buildExecutorService(BlockingQueue blockingQueue, String threadNamePrefix) { +- return new ThreadPoolExecutor( ++ return ThreadUtils.newThreadPoolExecutor( + this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(), + this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(), + 1000 * 60, +diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java +index 2dd3fc5b5..489d7b4fb 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java ++++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java +@@ -17,7 +17,6 @@ + package org.apache.rocketmq.store.stats; + + import java.util.HashMap; +-import java.util.concurrent.Executors; + import java.util.concurrent.ScheduledExecutorService; + import org.apache.commons.lang3.tuple.Pair; + import org.apache.rocketmq.common.BrokerConfig; +@@ -32,13 +31,14 @@ import org.apache.rocketmq.common.statistics.StatisticsItemScheduledPrinter; + import org.apache.rocketmq.common.statistics.StatisticsItemStateGetter; + import org.apache.rocketmq.common.statistics.StatisticsKindMeta; + import org.apache.rocketmq.common.statistics.StatisticsManager; ++import org.apache.rocketmq.common.stats.MomentStatsItemSet; + import org.apache.rocketmq.common.stats.Stats; ++import org.apache.rocketmq.common.stats.StatsItem; ++import org.apache.rocketmq.common.stats.StatsItemSet; + import org.apache.rocketmq.common.topic.TopicValidator; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +-import org.apache.rocketmq.common.stats.MomentStatsItemSet; +-import org.apache.rocketmq.common.stats.StatsItem; +-import org.apache.rocketmq.common.stats.StatsItemSet; + + public class BrokerStatsManager { + +@@ -281,11 +281,11 @@ public class BrokerStatsManager { + + private void initScheduleService() { + this.scheduledExecutorService = +- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig)); ++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig)); + this.commercialExecutor = +- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig)); ++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig)); + this.accountExecutor = +- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig)); ++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig)); + } + + public MomentStatsItemSet getMomentStatsItemSetFallSize() { +diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +index 181f7087a..0d50de65a 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +@@ -35,7 +35,6 @@ import java.util.Set; + import java.util.concurrent.BlockingQueue; + import java.util.concurrent.ConcurrentSkipListSet; + import java.util.concurrent.CountDownLatch; +-import java.util.concurrent.Executors; + import java.util.concurrent.LinkedBlockingDeque; + import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.TimeUnit; +@@ -54,6 +53,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.message.MessageExtBrokerInner; + import org.apache.rocketmq.common.topic.TopicValidator; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.store.ConsumeQueue; +@@ -174,11 +174,11 @@ public class TimerMessageStore { + this.lastBrokerRole = storeConfig.getBrokerRole(); + + if (messageStore instanceof DefaultMessageStore) { +- scheduler = Executors.newSingleThreadScheduledExecutor( ++ scheduler = ThreadUtils.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("TimerScheduledThread", + ((DefaultMessageStore) messageStore).getBrokerIdentity())); + } else { +- scheduler = Executors.newSingleThreadScheduledExecutor( ++ scheduler = ThreadUtils.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("TimerScheduledThread")); + } + +diff --git a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java +index f3d105bc6..080b7e385 100644 +--- a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java ++++ b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java +@@ -28,7 +28,6 @@ import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicInteger; + import java.util.concurrent.atomic.AtomicLong; +- + import javax.annotation.Generated; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java +index 6dd0e8846..65d586f43 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java +@@ -20,10 +20,10 @@ import java.util.concurrent.BlockingQueue; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledThreadPoolExecutor; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.common.ThreadFactoryImpl; ++import org.apache.rocketmq.common.utils.ThreadUtils; + + public class TieredStoreExecutor { + +@@ -43,20 +43,20 @@ public class TieredStoreExecutor { + public static ExecutorService compactIndexFileExecutor; + + public static void init() { +- commonScheduledExecutor = new ScheduledThreadPoolExecutor( ++ commonScheduledExecutor = ThreadUtils.newScheduledThreadPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), + new ThreadFactoryImpl("TieredCommonExecutor_")); + +- commitExecutor = new ScheduledThreadPoolExecutor( ++ commitExecutor = ThreadUtils.newScheduledThreadPool( + Math.max(16, Runtime.getRuntime().availableProcessors() * 4), + new ThreadFactoryImpl("TieredCommitExecutor_")); + +- cleanExpiredFileExecutor = new ScheduledThreadPoolExecutor( ++ cleanExpiredFileExecutor = ThreadUtils.newScheduledThreadPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), + new ThreadFactoryImpl("TieredCleanFileExecutor_")); + + dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); +- dispatchExecutor = new ThreadPoolExecutor( ++ dispatchExecutor = ThreadUtils.newThreadPoolExecutor( + Math.max(2, Runtime.getRuntime().availableProcessors()), + Math.max(16, Runtime.getRuntime().availableProcessors() * 4), + 1000 * 60, +@@ -66,7 +66,7 @@ public class TieredStoreExecutor { + new ThreadPoolExecutor.DiscardOldestPolicy()); + + fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); +- fetchDataExecutor = new ThreadPoolExecutor( ++ fetchDataExecutor = ThreadUtils.newThreadPoolExecutor( + Math.max(16, Runtime.getRuntime().availableProcessors() * 4), + Math.max(64, Runtime.getRuntime().availableProcessors() * 8), + 1000 * 60, +@@ -75,7 +75,7 @@ public class TieredStoreExecutor { + new ThreadFactoryImpl("TieredFetchExecutor_")); + + compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); +- compactIndexFileExecutor = new ThreadPoolExecutor( ++ compactIndexFileExecutor = ThreadUtils.newThreadPoolExecutor( + 1, + 1, + 1000 * 60, +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +index fa3596d51..1ebff6d8a 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +@@ -66,6 +66,7 @@ import org.apache.rocketmq.common.namesrv.NamesrvUtil; + import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.common.utils.NetworkUtil; + import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.RPCHook; +@@ -193,7 +194,7 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { + + int threadPoolCoreSize = Integer.parseInt(System.getProperty("rocketmq.admin.threadpool.coresize", "20")); + +- this.threadPoolExecutor = new ThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_")); ++ this.threadPoolExecutor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_")); + + break; + case RUNNING: +-- +2.32.0.windows.2 + + +From dad6b4dadfec7a58e78a6715ec16c2eb6b17ff27 Mon Sep 17 00:00:00 2001 +From: Ziyi Tan +Date: Mon, 11 Sep 2023 14:34:10 +0800 +Subject: [PATCH 2/6] [ISSUE #7334] `registerIncrementBrokerData` for single + topic update (#7335) + +Signed-off-by: Ziy1-Tan +--- + .../broker/topic/TopicConfigManager.java | 30 +++++++++++++++---- + 1 file changed, 25 insertions(+), 5 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +index 4e3c1736c..754605438 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +@@ -290,7 +290,11 @@ public class TopicConfigManager extends ConfigManager { + } + + if (createNew) { +- this.brokerController.registerBrokerAll(false, true, true); ++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++ this.brokerController.registerSingleTopicAll(topicConfig); ++ } else { ++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++ } + } + + return topicConfig; +@@ -394,7 +398,11 @@ public class TopicConfigManager extends ConfigManager { + } + + if (createNew) { +- this.brokerController.registerBrokerAll(false, true, true); ++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++ this.brokerController.registerSingleTopicAll(topicConfig); ++ } else { ++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++ } + } + + return topicConfig; +@@ -435,7 +443,11 @@ public class TopicConfigManager extends ConfigManager { + } + + if (createNew) { +- this.brokerController.registerBrokerAll(false, true, true); ++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++ this.brokerController.registerSingleTopicAll(topicConfig); ++ } else { ++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++ } + } + + return topicConfig; +@@ -461,7 +473,11 @@ public class TopicConfigManager extends ConfigManager { + dataVersion.nextVersion(stateMachineVersion); + + this.persist(); +- this.brokerController.registerBrokerAll(false, true, true); ++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++ this.brokerController.registerSingleTopicAll(topicConfig); ++ } else { ++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++ } + } + } + +@@ -484,7 +500,11 @@ public class TopicConfigManager extends ConfigManager { + dataVersion.nextVersion(stateMachineVersion); + + this.persist(); +- this.brokerController.registerBrokerAll(false, true, true); ++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++ this.brokerController.registerSingleTopicAll(topicConfig); ++ } else { ++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++ } + } + } + +-- +2.32.0.windows.2 + + +From 0dbd0772b99f618f757d42cd64542b83e2100e4f Mon Sep 17 00:00:00 2001 +From: Ziyi Tan +Date: Mon, 11 Sep 2023 15:48:07 +0800 +Subject: [PATCH 3/6] [ISSUE #7326] Split the request to register to the + nameserver (#7325) + +Signed-off-by: Ziy1-Tan +--- + .../rocketmq/broker/BrokerController.java | 41 +++++++++++-------- + .../broker/topic/TopicConfigManager.java | 21 ++++++++++ + .../apache/rocketmq/common/BrokerConfig.java | 24 +++++++++++ + .../test/route/CreateAndUpdateTopicIT.java | 31 ++++++++++++++ + 4 files changed, 99 insertions(+), 18 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index 275b64b1a..9e49f636d 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -1765,29 +1765,34 @@ public class BrokerController { + } + + public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) { ++ ConcurrentMap topicConfigMap = this.getTopicConfigManager().getTopicConfigTable(); ++ ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); + +- TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper(); +- +- topicConfigWrapper.setDataVersion(this.getTopicConfigManager().getDataVersion()); +- topicConfigWrapper.setTopicConfigTable(this.getTopicConfigManager().getTopicConfigTable()); +- +- topicConfigWrapper.setTopicQueueMappingInfoMap(this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream().map( +- entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue())) +- ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); +- +- if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) +- || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { +- ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); +- for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) { +- TopicConfig tmp = ++ for (TopicConfig topicConfig : topicConfigMap.values()) { ++ if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) ++ || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { ++ topicConfigTable.put(topicConfig.getTopicName(), + new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), +- topicConfig.getPerm() & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); +- topicConfigTable.put(topicConfig.getTopicName(), tmp); ++ topicConfig.getPerm() & getBrokerConfig().getBrokerPermission())); ++ } else { ++ topicConfigTable.put(topicConfig.getTopicName(), topicConfig); ++ } ++ ++ if (this.brokerConfig.isEnableSplitRegistration() ++ && topicConfigTable.size() >= this.brokerConfig.getSplitRegistrationSize()) { ++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildSerializeWrapper(topicConfigTable); ++ doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper); ++ topicConfigTable.clear(); + } +- topicConfigWrapper.setTopicConfigTable(topicConfigTable); + } + +- if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), ++ Map topicQueueMappingInfoMap = this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream() ++ .map(entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue()))) ++ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); ++ ++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager(). ++ buildSerializeWrapper(topicConfigTable, topicQueueMappingInfoMap); ++ if (this.brokerConfig.isEnableSplitRegistration() || forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), + this.getBrokerAddr(), + this.brokerConfig.getBrokerName(), + this.brokerConfig.getBrokerId(), +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +index 754605438..8537929be 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +@@ -29,6 +29,7 @@ import java.util.concurrent.locks.ReentrantLock; + + import com.google.common.collect.ImmutableMap; + ++import com.google.common.collect.Maps; + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.broker.BrokerController; + import org.apache.rocketmq.broker.BrokerPathConfigHelper; +@@ -47,7 +48,9 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.protocol.DataVersion; + import org.apache.rocketmq.remoting.protocol.body.KVTable; ++import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper; + import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; ++import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo; + + import static com.google.common.base.Preconditions.checkNotNull; + +@@ -609,6 +612,24 @@ public class TopicConfigManager extends ConfigManager { + return topicConfigSerializeWrapper; + } + ++ public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper(final ConcurrentMap topicConfigTable) { ++ return buildSerializeWrapper(topicConfigTable, Maps.newHashMap()); ++ } ++ ++ public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper( ++ final ConcurrentMap topicConfigTable, ++ final Map topicQueueMappingInfoMap ++ ) { ++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper(); ++ topicConfigWrapper.setTopicConfigTable(topicConfigTable); ++ topicConfigWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap); ++ topicConfigWrapper.setDataVersion(this.getDataVersion()); ++ if (this.brokerController.getBrokerConfig().isEnableSplitRegistration()) { ++ this.getDataVersion().nextVersion(); ++ } ++ return topicConfigWrapper; ++ } ++ + @Override + public String encode() { + return encode(false); +diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +index 45d26b29c..0d248c4e1 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +@@ -396,6 +396,14 @@ public class BrokerConfig extends BrokerIdentity { + + private boolean enableMixedMessageType = false; + ++ /** ++ * This flag and deleteTopicWithBrokerRegistration flag in the NameServer cannot be set to true at the same time, ++ * otherwise there will be a loss of routing ++ */ ++ private boolean enableSplitRegistration = false; ++ ++ private int splitRegistrationSize = 800; ++ + public long getMaxPopPollingSize() { + return maxPopPollingSize; + } +@@ -1731,4 +1739,20 @@ public class BrokerConfig extends BrokerIdentity { + public void setEnableMixedMessageType(boolean enableMixedMessageType) { + this.enableMixedMessageType = enableMixedMessageType; + } ++ ++ public boolean isEnableSplitRegistration() { ++ return enableSplitRegistration; ++ } ++ ++ public void setEnableSplitRegistration(boolean enableSplitRegistration) { ++ this.enableSplitRegistration = enableSplitRegistration; ++ } ++ ++ public int getSplitRegistrationSize() { ++ return splitRegistrationSize; ++ } ++ ++ public void setSplitRegistrationSize(int splitRegistrationSize) { ++ this.splitRegistrationSize = splitRegistrationSize; ++ } + } +diff --git a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java +index 7e3c7b871..2370e68c0 100644 +--- a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java ++++ b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java +@@ -17,6 +17,7 @@ + + package org.apache.rocketmq.test.route; + ++import org.apache.rocketmq.common.TopicConfig; + import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; + import org.apache.rocketmq.test.base.BaseConf; + import org.apache.rocketmq.test.util.MQAdminTestUtils; +@@ -111,4 +112,34 @@ public class CreateAndUpdateTopicIT extends BaseConf { + brokerController3.getBrokerConfig().setEnableSingleTopicRegister(false); + namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(false); + } ++ ++ @Test ++ public void testCreateOrUpdateTopic_EnableSplitRegistration() { ++ brokerController1.getBrokerConfig().setEnableSplitRegistration(true); ++ brokerController2.getBrokerConfig().setEnableSplitRegistration(true); ++ brokerController3.getBrokerConfig().setEnableSplitRegistration(true); ++ ++ String testTopic = "test-topic-"; ++ ++ for (int i = 0; i < 1000; i++) { ++ TopicConfig topicConfig = new TopicConfig(testTopic + i, 8, 8); ++ brokerController1.getTopicConfigManager().updateTopicConfig(topicConfig); ++ brokerController2.getTopicConfigManager().updateTopicConfig(topicConfig); ++ brokerController3.getTopicConfigManager().updateTopicConfig(topicConfig); ++ } ++ ++ brokerController1.registerBrokerAll(false, true, true); ++ brokerController2.registerBrokerAll(false, true, true); ++ brokerController3.registerBrokerAll(false, true, true); ++ ++ for (int i = 0; i < 1000; i++) { ++ TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic + i); ++ assertThat(route.getBrokerDatas()).hasSize(3); ++ assertThat(route.getQueueDatas()).hasSize(3); ++ } ++ ++ brokerController1.getBrokerConfig().setEnableSplitRegistration(false); ++ brokerController2.getBrokerConfig().setEnableSplitRegistration(false); ++ brokerController3.getBrokerConfig().setEnableSplitRegistration(false); ++ } + } +-- +2.32.0.windows.2 + + +From a9e353285cea762b0c5eab567bdfa8e5c8c2d279 Mon Sep 17 00:00:00 2001 +From: rongtong +Date: Mon, 11 Sep 2023 15:55:18 +0800 +Subject: [PATCH 4/6] Add the configuration of topicQueueLock number to better + support different scenarios (#7317) + +--- + .../main/java/org/apache/rocketmq/store/CommitLog.java | 2 +- + .../java/org/apache/rocketmq/store/TopicQueueLock.java | 8 ++++++++ + .../rocketmq/store/config/MessageStoreConfig.java | 10 ++++++++++ + 3 files changed, 19 insertions(+), 1 deletion(-) + +diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +index e6ee3bacc..456bf2b86 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +@@ -122,7 +122,7 @@ public class CommitLog implements Swappable { + + this.flushDiskWatcher = new FlushDiskWatcher(); + +- this.topicQueueLock = new TopicQueueLock(); ++ this.topicQueueLock = new TopicQueueLock(messageStore.getMessageStoreConfig().getTopicQueueLockNum()); + + this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java +index a78eeed23..5a131b5c3 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java ++++ b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java +@@ -34,6 +34,14 @@ public class TopicQueueLock { + } + } + ++ public TopicQueueLock(int size) { ++ this.size = size; ++ this.lockList = new ArrayList<>(size); ++ for (int i = 0; i < this.size; i++) { ++ this.lockList.add(new ReentrantLock()); ++ } ++ } ++ + public void lock(String topicQueueKey) { + Lock lock = this.lockList.get((topicQueueKey.hashCode() & 0x7fffffff) % this.size); + lock.lock(); +diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +index efb728ac0..9fa448043 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java ++++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +@@ -401,6 +401,8 @@ public class MessageStoreConfig { + private long memTableFlushInterval = 60 * 60 * 1000L; + private boolean enableRocksDBLog = false; + ++ private int topicQueueLockNum = 32; ++ + public boolean isDebugLockEnable() { + return debugLockEnable; + } +@@ -1751,4 +1753,12 @@ public class MessageStoreConfig { + public void setEnableRocksDBLog(boolean enableRocksDBLog) { + this.enableRocksDBLog = enableRocksDBLog; + } ++ ++ public int getTopicQueueLockNum() { ++ return topicQueueLockNum; ++ } ++ ++ public void setTopicQueueLockNum(int topicQueueLockNum) { ++ this.topicQueueLockNum = topicQueueLockNum; ++ } + } +-- +2.32.0.windows.2 + + +From 57f04c95d3a2ba6b91583058a6e4eda209f72d6e Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Mon, 11 Sep 2023 18:23:25 +0800 +Subject: [PATCH 5/6] [ISSUE #7343] Rollback modifications to registerProcessor + +Co-authored-by: guyinyou +--- + .../java/org/apache/rocketmq/broker/BrokerController.java | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index 9e49f636d..13a3feb4e 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -827,6 +827,8 @@ public class BrokerController { + + initializeResources(); + ++ registerProcessor(); ++ + initializeScheduledTasks(); + + initialTransaction(); +@@ -1687,8 +1689,6 @@ public class BrokerController { + } + } + }, 10, 5, TimeUnit.SECONDS); +- +- registerProcessor(); + } + + protected void scheduleSendHeartbeat() { +-- +2.32.0.windows.2 + + +From dad6ad09d13dadc36b6342671c77f619bbb8c522 Mon Sep 17 00:00:00 2001 +From: Ao Qiao +Date: Tue, 12 Sep 2023 08:28:45 +0800 +Subject: [PATCH 6/6] [ISSUE #7340] Abstract Duplicate code into a method in + `TopicConfigManager` (#7341) + +--- + .../broker/topic/TopicConfigManager.java | 44 ++++++------------- + 1 file changed, 14 insertions(+), 30 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +index 8537929be..511d29e12 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +@@ -293,11 +293,7 @@ public class TopicConfigManager extends ConfigManager { + } + + if (createNew) { +- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +- this.brokerController.registerSingleTopicAll(topicConfig); +- } else { +- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +- } ++ registerBrokerData(topicConfig); + } + + return topicConfig; +@@ -337,11 +333,7 @@ public class TopicConfigManager extends ConfigManager { + log.error("createTopicIfAbsent ", e); + } + if (createNew && register) { +- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +- this.brokerController.registerSingleTopicAll(topicConfig); +- } else { +- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +- } ++ registerBrokerData(topicConfig); + } + return getTopicConfig(topicConfig.getTopicName()); + } +@@ -401,11 +393,7 @@ public class TopicConfigManager extends ConfigManager { + } + + if (createNew) { +- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +- this.brokerController.registerSingleTopicAll(topicConfig); +- } else { +- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +- } ++ registerBrokerData(topicConfig); + } + + return topicConfig; +@@ -446,11 +434,7 @@ public class TopicConfigManager extends ConfigManager { + } + + if (createNew) { +- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +- this.brokerController.registerSingleTopicAll(topicConfig); +- } else { +- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +- } ++ registerBrokerData(topicConfig); + } + + return topicConfig; +@@ -476,11 +460,7 @@ public class TopicConfigManager extends ConfigManager { + dataVersion.nextVersion(stateMachineVersion); + + this.persist(); +- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +- this.brokerController.registerSingleTopicAll(topicConfig); +- } else { +- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +- } ++ registerBrokerData(topicConfig); + } + } + +@@ -503,11 +483,7 @@ public class TopicConfigManager extends ConfigManager { + dataVersion.nextVersion(stateMachineVersion); + + this.persist(); +- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +- this.brokerController.registerSingleTopicAll(topicConfig); +- } else { +- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +- } ++ registerBrokerData(topicConfig); + } + } + +@@ -699,6 +675,14 @@ public class TopicConfigManager extends ConfigManager { + } + } + ++ private void registerBrokerData(TopicConfig topicConfig) { ++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++ this.brokerController.registerSingleTopicAll(topicConfig); ++ } else { ++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++ } ++ } ++ + public boolean containsTopic(String topic) { + return topicConfigTable.containsKey(topic); + } +-- +2.32.0.windows.2 + diff --git a/patch018-backport-enhancement-of-tiered-storage.patch b/patch018-backport-enhancement-of-tiered-storage.patch new file mode 100644 index 000000000..ed79df9c7 --- /dev/null +++ b/patch018-backport-enhancement-of-tiered-storage.patch @@ -0,0 +1,601 @@ +From 1a8e7cb17cb29ed33b0196b52e452a6e76ade781 Mon Sep 17 00:00:00 2001 +From: yuz10 <845238369@qq.com> +Date: Tue, 12 Sep 2023 19:33:41 +0800 +Subject: [PATCH 1/5] [ISSUE #7345] Fix wrong result of searchOffset in tiered + storage + +--- + .../tieredstore/file/TieredFlatFile.java | 5 +- + .../tieredstore/file/TieredFlatFileTest.java | 46 +++++++++++++++++-- + 2 files changed, 46 insertions(+), 5 deletions(-) + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +index 426c4e09d..d973179ee 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +@@ -365,7 +365,10 @@ public class TieredFlatFile { + if (!segmentList.isEmpty()) { + return boundaryType == BoundaryType.UPPER ? segmentList.get(0) : segmentList.get(segmentList.size() - 1); + } +- return fileSegmentList.isEmpty() ? null : fileSegmentList.get(fileSegmentList.size() - 1); ++ if (fileSegmentList.isEmpty()) { ++ return null; ++ } ++ return boundaryType == BoundaryType.UPPER ? fileSegmentList.get(fileSegmentList.size() - 1) : fileSegmentList.get(0); + } finally { + fileSegmentLock.readLock().unlock(); + } +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java +index 7a4d05969..7e2fbf201 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java +@@ -16,10 +16,7 @@ + */ + package org.apache.rocketmq.tieredstore.file; + +-import java.io.IOException; +-import java.nio.ByteBuffer; +-import java.util.ArrayList; +-import java.util.List; ++import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; + import org.apache.rocketmq.tieredstore.common.FileSegmentType; +@@ -35,6 +32,11 @@ import org.junit.Assert; + import org.junit.Before; + import org.junit.Test; + ++import java.io.IOException; ++import java.nio.ByteBuffer; ++import java.util.ArrayList; ++import java.util.List; ++ + public class TieredFlatFileTest { + + private final String storePath = TieredStoreTestUtil.getRandomStorePath(); +@@ -301,4 +303,40 @@ public class TieredFlatFileTest { + fileQueue.rollingNewFile(); + Assert.assertEquals(2, fileQueue.getFileSegmentCount()); + } ++ ++ @Test ++ public void testGetFileByTime() { ++ String filePath = TieredStoreUtil.toPath(queue); ++ TieredFlatFile tieredFlatFile = fileQueueFactory.createFlatFileForCommitLog(filePath); ++ TieredFileSegment fileSegment1 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig); ++ fileSegment1.setMinTimestamp(100); ++ fileSegment1.setMaxTimestamp(200); ++ ++ TieredFileSegment fileSegment2 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig); ++ fileSegment2.setMinTimestamp(200); ++ fileSegment2.setMaxTimestamp(300); ++ ++ tieredFlatFile.getFileSegmentList().add(fileSegment1); ++ tieredFlatFile.getFileSegmentList().add(fileSegment2); ++ ++ TieredFileSegment segmentUpper = tieredFlatFile.getFileByTime(400, BoundaryType.UPPER); ++ Assert.assertEquals(fileSegment2, segmentUpper); ++ ++ TieredFileSegment segmentLower = tieredFlatFile.getFileByTime(400, BoundaryType.LOWER); ++ Assert.assertEquals(fileSegment2, segmentLower); ++ ++ ++ TieredFileSegment segmentUpper2 = tieredFlatFile.getFileByTime(0, BoundaryType.UPPER); ++ Assert.assertEquals(fileSegment1, segmentUpper2); ++ ++ TieredFileSegment segmentLower2 = tieredFlatFile.getFileByTime(0, BoundaryType.LOWER); ++ Assert.assertEquals(fileSegment1, segmentLower2); ++ ++ ++ TieredFileSegment segmentUpper3 = tieredFlatFile.getFileByTime(200, BoundaryType.UPPER); ++ Assert.assertEquals(fileSegment1, segmentUpper3); ++ ++ TieredFileSegment segmentLower3 = tieredFlatFile.getFileByTime(200, BoundaryType.LOWER); ++ Assert.assertEquals(fileSegment2, segmentLower3); ++ } + } +-- +2.32.0.windows.2 + + +From fd32dae2ab59f86dd215eeec405bf4fa6212bcb3 Mon Sep 17 00:00:00 2001 +From: lizhimins <707364882@qq.com> +Date: Tue, 12 Sep 2023 19:58:08 +0800 +Subject: [PATCH 2/5] [ISSUE #6633] Not clear uninitialized files and fix + metadata recover (#7342) + +--- + .../tieredstore/file/TieredFlatFile.java | 53 +++++++------------ + .../file/TieredFlatFileManager.java | 10 ++-- + 2 files changed, 22 insertions(+), 41 deletions(-) + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +index d973179ee..d96eb6e8f 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +@@ -16,7 +16,6 @@ + */ + package org.apache.rocketmq.tieredstore.file; + +-import com.alibaba.fastjson.JSON; + import com.google.common.annotations.VisibleForTesting; + import java.nio.ByteBuffer; + import java.util.ArrayList; +@@ -25,13 +24,13 @@ import java.util.Comparator; + import java.util.HashSet; + import java.util.LinkedList; + import java.util.List; +-import java.util.Objects; + import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.CopyOnWriteArrayList; + import java.util.concurrent.locks.ReentrantReadWriteLock; + import java.util.stream.Collectors; + import javax.annotation.Nullable; ++import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.tieredstore.common.AppendResult; +@@ -43,7 +42,6 @@ import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore; + import org.apache.rocketmq.tieredstore.provider.FileSegmentAllocator; + import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; + import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; +-import org.apache.rocketmq.common.BoundaryType; + + public class TieredFlatFile { + +@@ -177,7 +175,10 @@ public class TieredFlatFile { + } + } + +- private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fileSegment) { ++ /** ++ * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full ++ */ ++ public void updateFileSegment(TieredFileSegment fileSegment) { + + FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( + this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); +@@ -186,45 +187,24 @@ public class TieredFlatFile { + if (metadata == null) { + metadata = new FileSegmentMetadata( + this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); +- metadata.setCreateTimestamp(fileSegment.getMinTimestamp()); +- metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); +- metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); +- if (fileSegment.isClosed()) { +- metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); +- } +- this.tieredMetadataStore.updateFileSegment(metadata); ++ metadata.setCreateTimestamp(System.currentTimeMillis()); + } +- return metadata; +- } +- +- /** +- * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full +- */ +- public void updateFileSegment(TieredFileSegment fileSegment) { +- FileSegmentMetadata segmentMetadata = getOrCreateFileSegmentMetadata(fileSegment); + +- if (segmentMetadata.getStatus() == FileSegmentMetadata.STATUS_NEW +- && fileSegment.isFull() +- && !fileSegment.needCommit()) { ++ metadata.setSize(fileSegment.getCommitPosition()); ++ metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); ++ metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); + +- segmentMetadata.markSealed(); ++ if (fileSegment.isFull() && !fileSegment.needCommit()) { ++ if (metadata.getStatus() == FileSegmentMetadata.STATUS_NEW) { ++ metadata.markSealed(); ++ } + } + + if (fileSegment.isClosed()) { +- segmentMetadata.setStatus(FileSegmentMetadata.STATUS_DELETED); ++ metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); + } + +- segmentMetadata.setSize(fileSegment.getCommitPosition()); +- segmentMetadata.setEndTimestamp(fileSegment.getMaxTimestamp()); +- +- FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( +- this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); +- +- if (!Objects.equals(metadata, segmentMetadata)) { +- this.tieredMetadataStore.updateFileSegment(segmentMetadata); +- logger.debug("TieredFlatFile#UpdateSegmentMetadata, filePath: {}, content: {}", +- segmentMetadata.getPath(), JSON.toJSONString(segmentMetadata)); +- } ++ this.tieredMetadataStore.updateFileSegment(metadata); + } + + private void checkAndFixFileSize() { +@@ -598,6 +578,9 @@ public class TieredFlatFile { + logger.error("TieredFlatFile#destroy: mark file segment: {} is deleted failed", fileSegment.getPath(), e); + } + fileSegment.destroyFile(); ++ if (!fileSegment.exists()) { ++ tieredMetadataStore.deleteFileSegment(filePath, fileType, fileSegment.getBaseOffset()); ++ } + } + fileSegmentList.clear(); + needCommitFileSegmentList.clear(); +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +index 7c744af3b..087ea8c9c 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +@@ -136,15 +136,13 @@ public class TieredFlatFileManager { + TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime()); + for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) { + TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> { +- flatFile.getCompositeFlatFileLock().lock(); + try { ++ flatFile.getCompositeFlatFileLock().lock(); + flatFile.cleanExpiredFile(expiredTimeStamp); + flatFile.destroyExpiredFile(); +- if (flatFile.getConsumeQueueBaseOffset() == -1) { +- logger.info("Clean flatFile because file not initialized, topic={}, queueId={}", +- flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId()); +- destroyCompositeFile(flatFile.getMessageQueue()); +- } ++ } catch (Throwable t) { ++ logger.error("Do Clean expired file error, topic={}, queueId={}", ++ flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), t); + } finally { + flatFile.getCompositeFlatFileLock().unlock(); + } +-- +2.32.0.windows.2 + + +From 4a8e0d5b851d1f9573cda79b7d2e42ee498809da Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Wed, 13 Sep 2023 16:08:03 +0800 +Subject: [PATCH 3/5] [ISSUE #7351] Allow mqadmin to operate slave nodes + +Co-authored-by: guyinyou +--- + .../processor/AdminBrokerProcessor.java | 12 -- + .../processor/AdminBrokerProcessorTest.java | 106 ------------------ + 2 files changed, 118 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +index 8fbcd3c94..9e48431be 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +@@ -406,9 +406,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); +- if (validateSlave(response)) { +- return response; +- } + final CreateTopicRequestHeader requestHeader = + (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class); + +@@ -519,9 +516,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); +- if (validateSlave(response)) { +- return response; +- } + DeleteTopicRequestHeader requestHeader = + (DeleteTopicRequestHeader) request.decodeCommandCustomHeader(DeleteTopicRequestHeader.class); + +@@ -1413,9 +1407,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); +- if (validateSlave(response)) { +- return response; +- } + + LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}", + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); +@@ -1480,9 +1471,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); +- if (validateSlave(response)) { +- return response; +- } + DeleteSubscriptionGroupRequestHeader requestHeader = + (DeleteSubscriptionGroupRequestHeader) request.decodeCommandCustomHeader(DeleteSubscriptionGroupRequestHeader.class); + +diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +index 9d17011b6..ec252cece 100644 +--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +@@ -76,7 +76,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfi + import org.apache.rocketmq.store.DefaultMessageStore; + import org.apache.rocketmq.store.MessageStore; + import org.apache.rocketmq.store.SelectMappedBufferResult; +-import org.apache.rocketmq.store.config.BrokerRole; + import org.apache.rocketmq.store.config.MessageStoreConfig; + import org.apache.rocketmq.store.logfile.DefaultMappedFile; + import org.apache.rocketmq.store.stats.BrokerStats; +@@ -250,32 +249,6 @@ public class AdminBrokerProcessorTest { + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + +- @Test +- public void testUpdateAndCreateTopicOnSlaveInRocksdb() throws Exception { +- if (notToBeExecuted()) { +- return; +- } +- initRocksdbTopicManager(); +- testUpdateAndCreateTopicOnSlave(); +- } +- +- @Test +- public void testUpdateAndCreateTopicOnSlave() throws Exception { +- // setup +- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); +- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); +- defaultMessageStore = mock(DefaultMessageStore.class); +- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); +- +- // test on slave +- String topic = "TEST_CREATE_TOPIC"; +- RemotingCommand request = buildCreateTopicRequest(topic); +- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); +- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); +- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + +- "please execute it from master broker."); +- } +- + @Test + public void testDeleteTopicInRocksdb() throws Exception { + if (notToBeExecuted()) { +@@ -301,31 +274,6 @@ public class AdminBrokerProcessorTest { + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + +- @Test +- public void testDeleteTopicOnSlaveInRocksdb() throws Exception { +- if (notToBeExecuted()) { +- return; +- } +- initRocksdbTopicManager(); +- testDeleteTopicOnSlave(); +- } +- +- @Test +- public void testDeleteTopicOnSlave() throws Exception { +- // setup +- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); +- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); +- defaultMessageStore = mock(DefaultMessageStore.class); +- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); +- +- String topic = "TEST_DELETE_TOPIC"; +- RemotingCommand request = buildDeleteTopicRequest(topic); +- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); +- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); +- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + +- "please execute it from master broker."); +- } +- + @Test + public void testDeleteWithPopRetryTopic() throws Exception { + String topic = "topicA"; +@@ -538,36 +486,6 @@ public class AdminBrokerProcessorTest { + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + +- @Test +- public void testUpdateAndCreateSubscriptionGroupOnSlaveInRocksdb() throws Exception { +- initRocksdbSubscriptionManager(); +- testUpdateAndCreateSubscriptionGroupOnSlave(); +- } +- +- @Test +- public void testUpdateAndCreateSubscriptionGroupOnSlave() throws RemotingCommandException { +- // Setup +- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); +- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); +- defaultMessageStore = mock(DefaultMessageStore.class); +- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); +- +- // Test +- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null); +- SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); +- subscriptionGroupConfig.setBrokerId(1); +- subscriptionGroupConfig.setGroupName("groupId"); +- subscriptionGroupConfig.setConsumeEnable(Boolean.TRUE); +- subscriptionGroupConfig.setConsumeBroadcastEnable(Boolean.TRUE); +- subscriptionGroupConfig.setRetryMaxTimes(111); +- subscriptionGroupConfig.setConsumeFromMinEnable(Boolean.TRUE); +- request.setBody(JSON.toJSON(subscriptionGroupConfig).toString().getBytes()); +- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); +- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); +- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + +- "please execute it from master broker."); +- } +- + @Test + public void testGetAllSubscriptionGroupInRocksdb() throws Exception { + initRocksdbSubscriptionManager(); +@@ -596,30 +514,6 @@ public class AdminBrokerProcessorTest { + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + +- @Test +- public void testDeleteSubscriptionGroupOnSlaveInRocksdb() throws Exception { +- initRocksdbSubscriptionManager(); +- testDeleteSubscriptionGroupOnSlave(); +- } +- +- @Test +- public void testDeleteSubscriptionGroupOnSlave() throws RemotingCommandException { +- // Setup +- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); +- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); +- defaultMessageStore = mock(DefaultMessageStore.class); +- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); +- +- // Test +- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, null); +- request.addExtField("groupName", "GID-Group-Name"); +- request.addExtField("removeOffset", "true"); +- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); +- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); +- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + +- "please execute it from master broker."); +- } +- + @Test + public void testGetTopicStatsInfo() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, null); +-- +2.32.0.windows.2 + + +From 831fcc76cd7cd362bb6c136c287c624bb7eaf40a Mon Sep 17 00:00:00 2001 +From: lizhimins <707364882@qq.com> +Date: Tue, 19 Sep 2023 10:04:04 +0800 +Subject: [PATCH 4/5] [ISSUE #7363] Fix get message from tiered storage return + incorrect next pull offset (#7365) + +--- + .../tieredstore/TieredMessageFetcher.java | 2 +- + .../tieredstore/TieredMessageStore.java | 29 ++++++++++--------- + .../tieredstore/TieredMessageStoreTest.java | 5 ++-- + 3 files changed, 20 insertions(+), 16 deletions(-) + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java +index 766ff64f6..c948fa3fa 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java +@@ -319,7 +319,7 @@ public class TieredMessageFetcher implements MessageStoreFetcher { + } + + // if cache is miss, immediately pull messages +- LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + ++ LOGGER.info("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + + "topic: {}, queue: {}, queue offset: {}, max message num: {}", + mq.getTopic(), mq.getQueueId(), queueOffset, maxCount); + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +index 9fb1b2f01..d7d13d61e 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +@@ -147,6 +147,11 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + public CompletableFuture getMessageAsync(String group, String topic, + int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { + ++ // For system topic, force reading from local store ++ if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { ++ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); ++ } ++ + if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) { + logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); + } else { +@@ -158,6 +163,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + return fetcher + .getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter) + .thenApply(result -> { ++ + Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder() + .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_GET_MESSAGE) + .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic) +@@ -166,8 +172,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); + + if (result.getStatus() == GetMessageStatus.OFFSET_FOUND_NULL || +- result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_ONE || +- result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_BADLY) { ++ result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { + + if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) { + TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes); +@@ -178,14 +183,8 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + } + } + +- // Fetch system topic data from the broker when using the force level. +- if (result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { +- if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { +- return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); +- } +- } +- + if (result.getStatus() != GetMessageStatus.FOUND && ++ result.getStatus() != GetMessageStatus.NO_MATCHED_LOGIC_QUEUE && + result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_ONE && + result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_BADLY) { + logger.warn("GetMessageAsync not found and message is not in next store, result: {}, " + +@@ -206,10 +205,14 @@ public class TieredMessageStore extends AbstractPluginMessageStore { + if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) { + result.setMinOffset(minOffsetInQueue); + } +- long maxOffsetInQueue = next.getMaxOffsetInQueue(topic, queueId); +- if (maxOffsetInQueue >= 0 && maxOffsetInQueue > result.getMaxOffset()) { +- result.setMaxOffset(maxOffsetInQueue); +- } ++ ++ // In general, the local cq offset is slightly greater than the commit offset in read message, ++ // so there is no need to update the maximum offset to the local cq offset here, ++ // otherwise it will cause repeated consumption after next begin offset over commit offset. ++ ++ logger.trace("GetMessageAsync result, group: {}, topic: {}, queueId: {}, offset: {}, count:{}, {}", ++ group, topic, queueId, offset, maxMsgNums, result); ++ + return result; + }).exceptionally(e -> { + logger.error("GetMessageAsync from tiered store failed", e); +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java +index 2451199c2..07af1fc8b 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java +@@ -168,7 +168,7 @@ public class TieredMessageStoreTest { + GetMessageResult result1 = new GetMessageResult(); + result1.setStatus(GetMessageStatus.FOUND); + GetMessageResult result2 = new GetMessageResult(); +- result2.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING); ++ result2.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY); + + when(fetcher.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(result1)); + when(nextStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(result2); +@@ -188,7 +188,8 @@ public class TieredMessageStoreTest { + properties.setProperty("tieredStorageLevel", "3"); + configuration.update(properties); + when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); +- Assert.assertSame(result2, store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null)); ++ Assert.assertEquals(result2.getStatus(), ++ store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null).getStatus()); + } + + @Test +-- +2.32.0.windows.2 + + +From f05a8da760dfade411ad56ef874f477988479cf9 Mon Sep 17 00:00:00 2001 +From: rongtong +Date: Wed, 20 Sep 2023 15:06:21 +0800 +Subject: [PATCH 5/5] Print admin queue watermark in log (#7372) + +--- + .../main/java/org/apache/rocketmq/broker/BrokerController.java | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index 13a3feb4e..53e2e1b62 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -1182,6 +1182,7 @@ public class BrokerController { + LOG_WATER_MARK.info("[WATERMARK] ClientManager Queue Size: {} SlowTimeMills: {}", this.clientManagerThreadPoolQueue.size(), this.headSlowTimeMills(this.clientManagerThreadPoolQueue)); + LOG_WATER_MARK.info("[WATERMARK] Heartbeat Queue Size: {} SlowTimeMills: {}", this.heartbeatThreadPoolQueue.size(), this.headSlowTimeMills(this.heartbeatThreadPoolQueue)); + LOG_WATER_MARK.info("[WATERMARK] Ack Queue Size: {} SlowTimeMills: {}", this.ackThreadPoolQueue.size(), headSlowTimeMills(this.ackThreadPoolQueue)); ++ LOG_WATER_MARK.info("[WATERMARK] Admin Queue Size: {} SlowTimeMills: {}", this.adminBrokerThreadPoolQueue.size(), headSlowTimeMills(this.adminBrokerThreadPoolQueue)); + } + + public MessageStore getMessageStore() { +-- +2.32.0.windows.2 + diff --git a/patch019-backport-some-bugfix.patch b/patch019-backport-some-bugfix.patch new file mode 100644 index 000000000..d85f01c98 --- /dev/null +++ b/patch019-backport-some-bugfix.patch @@ -0,0 +1,1499 @@ +From 42fcd278ca84f6988d48a7d11271fc81b921d59a Mon Sep 17 00:00:00 2001 +From: lk +Date: Wed, 20 Sep 2023 15:41:23 +0800 +Subject: [PATCH 01/12] [ISSUE #7374] Prepare to release Apache RocketMQ 5.1.4 + (#7375) + +--- + common/src/main/java/org/apache/rocketmq/common/MQVersion.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +index bfd07a895..4f1990ff8 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java ++++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +@@ -18,7 +18,7 @@ package org.apache.rocketmq.common; + + public class MQVersion { + +- public static final int CURRENT_VERSION = Version.V5_1_3.ordinal(); ++ public static final int CURRENT_VERSION = Version.V5_1_4.ordinal(); + + public static String getVersionDesc(int value) { + int length = Version.values().length; +-- +2.32.0.windows.2 + + +From b8610d87bb55de1f4413460c05da529dab60c1c1 Mon Sep 17 00:00:00 2001 +From: Jixiang Jin +Date: Thu, 21 Sep 2023 16:21:44 +0800 +Subject: [PATCH 02/12] Replace loggingMetricExporter with + OtlpJsonLoggingMetricExporter. (#7373) + +* Replace loggingMetricExporter with OtlpJsonLoggingMetricExporter. + +* Fix bazel workspace + +* Move OtlpJsonLoggingMetricExporter to proxy and controller. + +* Fix Bazel imports. +--- + WORKSPACE | 1 + + broker/BUILD.bazel | 1 + + .../rocketmq/broker/metrics/BrokerMetricsManager.java | 9 +++++---- + broker/src/main/resources/rmq.broker.logback.xml | 5 +++++ + common/BUILD.bazel | 1 + + common/pom.xml | 4 ++++ + controller/BUILD.bazel | 1 + + .../controller/metrics/ControllerMetricsManager.java | 9 +++++---- + pom.xml | 5 +++++ + proxy/BUILD.bazel | 1 + + .../rocketmq/proxy/metrics/ProxyMetricsManager.java | 11 ++++++----- + proxy/src/main/resources/rmq.proxy.logback.xml | 5 +++++ + tieredstore/BUILD.bazel | 1 + + 13 files changed, 41 insertions(+), 13 deletions(-) + +diff --git a/WORKSPACE b/WORKSPACE +index 3126f2d1d..8640485ba 100644 +--- a/WORKSPACE ++++ b/WORKSPACE +@@ -92,6 +92,7 @@ maven_install( + "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha", + "io.opentelemetry:opentelemetry-exporter-logging:1.29.0", + "io.opentelemetry:opentelemetry-sdk:1.29.0", ++ "io.opentelemetry:opentelemetry-exporter-logging-otlp:1.29.0", + "com.squareup.okio:okio-jvm:3.0.0", + "io.opentelemetry:opentelemetry-api:1.29.0", + "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0", +diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel +index 6adcdc7b9..64cb1b341 100644 +--- a/broker/BUILD.bazel ++++ b/broker/BUILD.bazel +@@ -44,6 +44,7 @@ java_library( + "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", + "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", + "@maven//:io_opentelemetry_opentelemetry_sdk", + "@maven//:io_opentelemetry_opentelemetry_sdk_common", + "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +index 6af5afc14..39af18b9f 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +@@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongCounter; + import io.opentelemetry.api.metrics.LongHistogram; + import io.opentelemetry.api.metrics.Meter; + import io.opentelemetry.api.metrics.ObservableLongGauge; +-import io.opentelemetry.exporter.logging.LoggingMetricExporter; ++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; + import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; + import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; + import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; +@@ -36,6 +36,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; + import io.opentelemetry.sdk.metrics.View; + import io.opentelemetry.sdk.metrics.ViewBuilder; + import io.opentelemetry.sdk.metrics.data.AggregationTemporality; ++import io.opentelemetry.sdk.metrics.export.MetricExporter; + import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; + import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; + import io.opentelemetry.sdk.resources.Resource; +@@ -113,7 +114,7 @@ public class BrokerMetricsManager { + private OtlpGrpcMetricExporter metricExporter; + private PeriodicMetricReader periodicMetricReader; + private PrometheusHttpServer prometheusHttpServer; +- private LoggingMetricExporter loggingMetricExporter; ++ private MetricExporter loggingMetricExporter; + private Meter brokerMeter; + + public static Supplier attributesBuilderSupplier = Attributes::builder; +@@ -327,8 +328,8 @@ public class BrokerMetricsManager { + if (metricsExporterType == MetricsExporterType.LOG) { + SLF4JBridgeHandler.removeHandlersForRootLogger(); + SLF4JBridgeHandler.install(); +- loggingMetricExporter = LoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); +- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); ++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); ++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); + periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) + .setInterval(brokerConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) + .build(); +diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml +index 3c51e59d4..32dc29736 100644 +--- a/broker/src/main/resources/rmq.broker.logback.xml ++++ b/broker/src/main/resources/rmq.broker.logback.xml +@@ -672,6 +672,11 @@ + + + ++ ++ ++ ++ ++ + + + +diff --git a/common/BUILD.bazel b/common/BUILD.bazel +index a95a19ccd..e6701d0fc 100644 +--- a/common/BUILD.bazel ++++ b/common/BUILD.bazel +@@ -35,6 +35,7 @@ java_library( + "@maven//:io_opentelemetry_opentelemetry_sdk", + "@maven//:io_opentelemetry_opentelemetry_sdk_common", + "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", + "@maven//:org_apache_commons_commons_lang3", + "@maven//:org_lz4_lz4_java", + "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", +diff --git a/common/pom.xml b/common/pom.xml +index 31eb0f087..accc7f0a8 100644 +--- a/common/pom.xml ++++ b/common/pom.xml +@@ -80,6 +80,10 @@ + io.opentelemetry + opentelemetry-sdk + ++ ++ io.opentelemetry ++ opentelemetry-exporter-logging-otlp ++ + + io.grpc + grpc-stub +diff --git a/controller/BUILD.bazel b/controller/BUILD.bazel +index 843d9dc77..b2b743eb2 100644 +--- a/controller/BUILD.bazel ++++ b/controller/BUILD.bazel +@@ -49,6 +49,7 @@ java_library( + "@maven//:io_opentelemetry_opentelemetry_sdk_common", + "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", + "@maven//:org_slf4j_jul_to_slf4j", + ], + ) +diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +index 650740bcc..be9e77eea 100644 +--- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +@@ -26,7 +26,7 @@ import io.opentelemetry.api.metrics.LongHistogram; + import io.opentelemetry.api.metrics.LongUpDownCounter; + import io.opentelemetry.api.metrics.Meter; + import io.opentelemetry.api.metrics.ObservableLongGauge; +-import io.opentelemetry.exporter.logging.LoggingMetricExporter; ++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; + import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; + import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; + import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; +@@ -38,6 +38,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProvider; + import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; + import io.opentelemetry.sdk.metrics.View; + import io.opentelemetry.sdk.metrics.data.AggregationTemporality; ++import io.opentelemetry.sdk.metrics.export.MetricExporter; + import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; + import io.opentelemetry.sdk.resources.Resource; + import java.io.File; +@@ -121,7 +122,7 @@ public class ControllerMetricsManager { + + private PrometheusHttpServer prometheusHttpServer; + +- private LoggingMetricExporter loggingMetricExporter; ++ private MetricExporter loggingMetricExporter; + + public static ControllerMetricsManager getInstance(ControllerManager controllerManager) { + if (instance == null) { +@@ -364,8 +365,8 @@ public class ControllerMetricsManager { + if (type == MetricsExporterType.LOG) { + SLF4JBridgeHandler.removeHandlersForRootLogger(); + SLF4JBridgeHandler.install(); +- loggingMetricExporter = LoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); +- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); ++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); ++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); + periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) + .setInterval(config.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) + .build(); +diff --git a/pom.xml b/pom.xml +index 9f0b3eb96..4b382c6da 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -974,6 +974,11 @@ + opentelemetry-sdk + ${opentelemetry.version} + ++ ++ io.opentelemetry ++ opentelemetry-exporter-logging-otlp ++ ${opentelemetry.version} ++ + + org.slf4j + jul-to-slf4j +diff --git a/proxy/BUILD.bazel b/proxy/BUILD.bazel +index b4f3c16e2..cb7af9254 100644 +--- a/proxy/BUILD.bazel ++++ b/proxy/BUILD.bazel +@@ -52,6 +52,7 @@ java_library( + "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", + "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", + "@maven//:io_opentelemetry_opentelemetry_sdk", + "@maven//:io_opentelemetry_opentelemetry_sdk_common", + "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java +index f5050858f..2b8dac5d8 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java +@@ -21,15 +21,16 @@ import io.opentelemetry.api.common.Attributes; + import io.opentelemetry.api.common.AttributesBuilder; + import io.opentelemetry.api.metrics.Meter; + import io.opentelemetry.api.metrics.ObservableLongGauge; ++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; + import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; + import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; + import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; +-import io.opentelemetry.exporter.logging.LoggingMetricExporter; + import io.opentelemetry.sdk.OpenTelemetrySdk; + import io.opentelemetry.sdk.metrics.InstrumentType; + import io.opentelemetry.sdk.metrics.SdkMeterProvider; + import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; + import io.opentelemetry.sdk.metrics.data.AggregationTemporality; ++import io.opentelemetry.sdk.metrics.export.MetricExporter; + import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; + import io.opentelemetry.sdk.resources.Resource; + import java.util.HashMap; +@@ -42,9 +43,9 @@ import org.apache.rocketmq.broker.metrics.BrokerMetricsManager; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.metrics.MetricsExporterType; + import org.apache.rocketmq.common.utils.StartAndShutdown; +-import org.apache.rocketmq.proxy.config.ProxyConfig; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.proxy.config.ProxyConfig; + import org.slf4j.bridge.SLF4JBridgeHandler; + + import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.AGGREGATION_DELTA; +@@ -67,7 +68,7 @@ public class ProxyMetricsManager implements StartAndShutdown { + private OtlpGrpcMetricExporter metricExporter; + private PeriodicMetricReader periodicMetricReader; + private PrometheusHttpServer prometheusHttpServer; +- private LoggingMetricExporter loggingMetricExporter; ++ private MetricExporter loggingMetricExporter; + + public static ObservableLongGauge proxyUp = null; + +@@ -221,8 +222,8 @@ public class ProxyMetricsManager implements StartAndShutdown { + if (metricsExporterType == MetricsExporterType.LOG) { + SLF4JBridgeHandler.removeHandlersForRootLogger(); + SLF4JBridgeHandler.install(); +- loggingMetricExporter = LoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); +- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); ++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); ++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); + periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) + .setInterval(proxyConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) + .build(); +diff --git a/proxy/src/main/resources/rmq.proxy.logback.xml b/proxy/src/main/resources/rmq.proxy.logback.xml +index d38827f92..f968a45e6 100644 +--- a/proxy/src/main/resources/rmq.proxy.logback.xml ++++ b/proxy/src/main/resources/rmq.proxy.logback.xml +@@ -418,6 +418,11 @@ + + + ++ ++ ++ ++ ++ + + + +diff --git a/tieredstore/BUILD.bazel b/tieredstore/BUILD.bazel +index 5b3885a4e..dea2c561b 100644 +--- a/tieredstore/BUILD.bazel ++++ b/tieredstore/BUILD.bazel +@@ -36,6 +36,7 @@ java_library( + "@maven//:io_opentelemetry_opentelemetry_sdk", + "@maven//:io_opentelemetry_opentelemetry_sdk_common", + "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", + "@maven//:org_apache_commons_commons_lang3", + "@maven//:org_apache_tomcat_annotations_api", + "@maven//:com_alibaba_fastjson", +-- +2.32.0.windows.2 + + +From 1a681bdf9b5c5ab0be446d6394c0cac8768f45d9 Mon Sep 17 00:00:00 2001 +From: lk +Date: Thu, 21 Sep 2023 19:58:29 +0800 +Subject: [PATCH 03/12] [maven-release-plugin] prepare release + rocketmq-all-5.1.4 (#7377) + +--- + acl/pom.xml | 2 +- + broker/pom.xml | 2 +- + client/pom.xml | 2 +- + common/pom.xml | 2 +- + container/pom.xml | 2 +- + controller/pom.xml | 2 +- + distribution/pom.xml | 2 +- + example/pom.xml | 2 +- + filter/pom.xml | 2 +- + namesrv/pom.xml | 2 +- + openmessaging/pom.xml | 2 +- + pom.xml | 4 ++-- + proxy/pom.xml | 2 +- + remoting/pom.xml | 2 +- + srvutil/pom.xml | 2 +- + store/pom.xml | 2 +- + test/pom.xml | 2 +- + tieredstore/pom.xml | 2 +- + tools/pom.xml | 2 +- + 19 files changed, 20 insertions(+), 20 deletions(-) + +diff --git a/acl/pom.xml b/acl/pom.xml +index 989c0cf77..9f6838b00 100644 +--- a/acl/pom.xml ++++ b/acl/pom.xml +@@ -13,7 +13,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + rocketmq-acl + rocketmq-acl ${project.version} +diff --git a/broker/pom.xml b/broker/pom.xml +index 16e026276..d483e67ba 100644 +--- a/broker/pom.xml ++++ b/broker/pom.xml +@@ -13,7 +13,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/client/pom.xml b/client/pom.xml +index c59a43889..4febedc6d 100644 +--- a/client/pom.xml ++++ b/client/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/common/pom.xml b/common/pom.xml +index accc7f0a8..b70873dfa 100644 +--- a/common/pom.xml ++++ b/common/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/container/pom.xml b/container/pom.xml +index c8499f127..e6c1f4b4d 100644 +--- a/container/pom.xml ++++ b/container/pom.xml +@@ -18,7 +18,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/controller/pom.xml b/controller/pom.xml +index 3346c7c82..46a3834c6 100644 +--- a/controller/pom.xml ++++ b/controller/pom.xml +@@ -19,7 +19,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4-SNAPSHOT ++ 5.1.4 + + 4.0.0 + jar +diff --git a/distribution/pom.xml b/distribution/pom.xml +index dbde2d9d4..346c4de35 100644 +--- a/distribution/pom.xml ++++ b/distribution/pom.xml +@@ -20,7 +20,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + rocketmq-distribution + rocketmq-distribution ${project.version} +diff --git a/example/pom.xml b/example/pom.xml +index 862fc3169..9e7db43f8 100644 +--- a/example/pom.xml ++++ b/example/pom.xml +@@ -19,7 +19,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/filter/pom.xml b/filter/pom.xml +index 3fe51ceae..84189066d 100644 +--- a/filter/pom.xml ++++ b/filter/pom.xml +@@ -20,7 +20,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/namesrv/pom.xml b/namesrv/pom.xml +index 684b2683c..7c218078a 100644 +--- a/namesrv/pom.xml ++++ b/namesrv/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml +index aaa4c896c..fd499e3de 100644 +--- a/openmessaging/pom.xml ++++ b/openmessaging/pom.xml +@@ -20,7 +20,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/pom.xml b/pom.xml +index 4b382c6da..0e1d04f15 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -28,7 +28,7 @@ + 2012 + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + pom + Apache RocketMQ ${project.version} + http://rocketmq.apache.org/ +@@ -37,7 +37,7 @@ + git@github.com:apache/rocketmq.git + scm:git:git@github.com:apache/rocketmq.git + scm:git:git@github.com:apache/rocketmq.git +- HEAD ++ rocketmq-all-5.1.4 + + + +diff --git a/proxy/pom.xml b/proxy/pom.xml +index 3fbea107a..abf242eee 100644 +--- a/proxy/pom.xml ++++ b/proxy/pom.xml +@@ -20,7 +20,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/remoting/pom.xml b/remoting/pom.xml +index 8a43c5c30..fc70cb62e 100644 +--- a/remoting/pom.xml ++++ b/remoting/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/srvutil/pom.xml b/srvutil/pom.xml +index fa54ad019..d7f946cce 100644 +--- a/srvutil/pom.xml ++++ b/srvutil/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/store/pom.xml b/store/pom.xml +index 38f04009d..6d6983c5d 100644 +--- a/store/pom.xml ++++ b/store/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/test/pom.xml b/test/pom.xml +index 8f25c35c9..39090e426 100644 +--- a/test/pom.xml ++++ b/test/pom.xml +@@ -20,7 +20,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml +index c476040ba..7b209751f 100644 +--- a/tieredstore/pom.xml ++++ b/tieredstore/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +diff --git a/tools/pom.xml b/tools/pom.xml +index 1c3b431bc..806787ec9 100644 +--- a/tools/pom.xml ++++ b/tools/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4-SNAPSHOT ++ 5.1.4 + + + 4.0.0 +-- +2.32.0.windows.2 + + +From 73b3fde83765e066541e3455cd1e6604292a9b7c Mon Sep 17 00:00:00 2001 +From: lk +Date: Fri, 22 Sep 2023 10:08:59 +0800 +Subject: [PATCH 04/12] [maven-release-plugin] prepare for next development + iteration (#7379) + +--- + acl/pom.xml | 2 +- + broker/pom.xml | 2 +- + client/pom.xml | 2 +- + common/pom.xml | 2 +- + container/pom.xml | 2 +- + controller/pom.xml | 2 +- + distribution/pom.xml | 2 +- + example/pom.xml | 2 +- + filter/pom.xml | 2 +- + namesrv/pom.xml | 2 +- + openmessaging/pom.xml | 2 +- + pom.xml | 4 ++-- + proxy/pom.xml | 2 +- + remoting/pom.xml | 2 +- + srvutil/pom.xml | 2 +- + store/pom.xml | 2 +- + test/pom.xml | 2 +- + tieredstore/pom.xml | 2 +- + tools/pom.xml | 2 +- + 19 files changed, 20 insertions(+), 20 deletions(-) + +diff --git a/acl/pom.xml b/acl/pom.xml +index 9f6838b00..8a296e5ae 100644 +--- a/acl/pom.xml ++++ b/acl/pom.xml +@@ -13,7 +13,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + rocketmq-acl + rocketmq-acl ${project.version} +diff --git a/broker/pom.xml b/broker/pom.xml +index d483e67ba..add83045d 100644 +--- a/broker/pom.xml ++++ b/broker/pom.xml +@@ -13,7 +13,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/client/pom.xml b/client/pom.xml +index 4febedc6d..d6fb3889b 100644 +--- a/client/pom.xml ++++ b/client/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/common/pom.xml b/common/pom.xml +index b70873dfa..6104c3ac6 100644 +--- a/common/pom.xml ++++ b/common/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/container/pom.xml b/container/pom.xml +index e6c1f4b4d..8af231e01 100644 +--- a/container/pom.xml ++++ b/container/pom.xml +@@ -18,7 +18,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/controller/pom.xml b/controller/pom.xml +index 46a3834c6..8432b220b 100644 +--- a/controller/pom.xml ++++ b/controller/pom.xml +@@ -19,7 +19,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4 ++ 5.1.5-SNAPSHOT + + 4.0.0 + jar +diff --git a/distribution/pom.xml b/distribution/pom.xml +index 346c4de35..73474d34a 100644 +--- a/distribution/pom.xml ++++ b/distribution/pom.xml +@@ -20,7 +20,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + rocketmq-distribution + rocketmq-distribution ${project.version} +diff --git a/example/pom.xml b/example/pom.xml +index 9e7db43f8..a8c7f5382 100644 +--- a/example/pom.xml ++++ b/example/pom.xml +@@ -19,7 +19,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/filter/pom.xml b/filter/pom.xml +index 84189066d..892f46e9d 100644 +--- a/filter/pom.xml ++++ b/filter/pom.xml +@@ -20,7 +20,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/namesrv/pom.xml b/namesrv/pom.xml +index 7c218078a..e320ed573 100644 +--- a/namesrv/pom.xml ++++ b/namesrv/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml +index fd499e3de..f10c8af6f 100644 +--- a/openmessaging/pom.xml ++++ b/openmessaging/pom.xml +@@ -20,7 +20,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/pom.xml b/pom.xml +index 0e1d04f15..4202d4095 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -28,7 +28,7 @@ + 2012 + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + pom + Apache RocketMQ ${project.version} + http://rocketmq.apache.org/ +@@ -37,7 +37,7 @@ + git@github.com:apache/rocketmq.git + scm:git:git@github.com:apache/rocketmq.git + scm:git:git@github.com:apache/rocketmq.git +- rocketmq-all-5.1.4 ++ HEAD + + + +diff --git a/proxy/pom.xml b/proxy/pom.xml +index abf242eee..5c5349a8c 100644 +--- a/proxy/pom.xml ++++ b/proxy/pom.xml +@@ -20,7 +20,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/remoting/pom.xml b/remoting/pom.xml +index fc70cb62e..f78480680 100644 +--- a/remoting/pom.xml ++++ b/remoting/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/srvutil/pom.xml b/srvutil/pom.xml +index d7f946cce..894e9cc6f 100644 +--- a/srvutil/pom.xml ++++ b/srvutil/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/store/pom.xml b/store/pom.xml +index 6d6983c5d..e979030e8 100644 +--- a/store/pom.xml ++++ b/store/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/test/pom.xml b/test/pom.xml +index 39090e426..168cbab0b 100644 +--- a/test/pom.xml ++++ b/test/pom.xml +@@ -20,7 +20,7 @@ + + rocketmq-all + org.apache.rocketmq +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml +index 7b209751f..b2ea40bf3 100644 +--- a/tieredstore/pom.xml ++++ b/tieredstore/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +diff --git a/tools/pom.xml b/tools/pom.xml +index 806787ec9..e1daa57a6 100644 +--- a/tools/pom.xml ++++ b/tools/pom.xml +@@ -19,7 +19,7 @@ + + org.apache.rocketmq + rocketmq-all +- 5.1.4 ++ 5.1.5-SNAPSHOT + + + 4.0.0 +-- +2.32.0.windows.2 + + +From 88a9d939ce110381b3b418370d4711c0c214dc7f Mon Sep 17 00:00:00 2001 +From: Ji Juntao +Date: Sat, 23 Sep 2023 17:38:27 +0800 +Subject: [PATCH 05/12] [ISSUE #7381] Fix the problem of inaccurate timer + message metric (#7382) + +* correct the timerMetrics' result. + +* for further extension. + +* checkstyle. + +* use toLong. +--- + .../store/timer/TimerMessageStore.java | 20 +++++++++++++++---- + .../rocketmq/store/timer/TimerMetrics.java | 5 ++++- + .../rocketmq/store/timer/TimerRequest.java | 7 +++++-- + .../store/timer/TimerMetricsTest.java | 10 ++++++++-- + 4 files changed, 33 insertions(+), 9 deletions(-) + +diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +index 0d50de65a..ac4c61cd6 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +@@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicInteger; + import java.util.function.Function; + import org.apache.commons.collections.CollectionUtils; ++import org.apache.commons.lang3.math.NumberUtils; + import org.apache.rocketmq.common.ServiceThread; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.TopicFilterType; +@@ -599,7 +600,12 @@ public class TimerMessageStore { + if (null == msg || null == msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC)) { + return; + } +- timerMetrics.addAndGet(msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC), value); ++ if (msg.getProperty(TIMER_ENQUEUE_MS) != null ++ && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) { ++ return; ++ } ++ // pass msg into addAndGet, for further more judgement extension. ++ timerMetrics.addAndGet(msg, value); + } catch (Throwable t) { + if (frequency.incrementAndGet() % 1000 == 0) { + LOGGER.error("error in adding metric", t); +@@ -1323,6 +1329,7 @@ public class TimerMessageStore { + perfCounterTicks.startTick(ENQUEUE_PUT); + DefaultStoreMetricsManager.incTimerEnqueueCount(getRealTopic(req.getMsg())); + if (shouldRunningDequeue && req.getDelayTime() < currWriteTimeMs) { ++ req.setEnqueueTime(Long.MAX_VALUE); + dequeuePutQueue.put(req); + } else { + boolean doEnqueueRes = doEnqueue( +@@ -1452,9 +1459,14 @@ public class TimerMessageStore { + } + try { + perfCounterTicks.startTick(DEQUEUE_PUT); +- DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(tr.getMsg())); +- addMetric(tr.getMsg(), -1); +- MessageExtBrokerInner msg = convert(tr.getMsg(), tr.getEnqueueTime(), needRoll(tr.getMagic())); ++ MessageExt msgExt = tr.getMsg(); ++ DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(msgExt)); ++ if (tr.getEnqueueTime() == Long.MAX_VALUE) { ++ // never enqueue, mark it. ++ MessageAccessor.putProperty(msgExt, TIMER_ENQUEUE_MS, String.valueOf(Long.MAX_VALUE)); ++ } ++ addMetric(msgExt, -1); ++ MessageExtBrokerInner msg = convert(msgExt, tr.getEnqueueTime(), needRoll(tr.getMagic())); + doRes = PUT_NEED_RETRY != doPut(msg, needRoll(tr.getMagic())); + while (!doRes && !isStopped()) { + if (!isRunningDequeue()) { +diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java +index e7b00cc07..7f8fedd8a 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java ++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java +@@ -38,6 +38,8 @@ import java.util.concurrent.locks.Lock; + import java.util.concurrent.locks.ReentrantLock; + import org.apache.rocketmq.common.ConfigManager; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.message.MessageConst; ++import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +@@ -78,7 +80,8 @@ public class TimerMetrics extends ConfigManager { + return distPair.getCount().addAndGet(value); + } + +- public long addAndGet(String topic, int value) { ++ public long addAndGet(MessageExt msg, int value) { ++ String topic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC); + Metric pair = getTopicPair(topic); + getDataVersion().nextVersion(); + pair.setTimeStamp(System.currentTimeMillis()); +diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java +index 1dd64f759..1b25d355c 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java ++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java +@@ -27,8 +27,9 @@ public class TimerRequest { + private final int sizePy; + private final long delayTime; + +- private final long enqueueTime; + private final int magic; ++ ++ private long enqueueTime; + private MessageExt msg; + + +@@ -94,7 +95,9 @@ public class TimerRequest { + public void setLatch(CountDownLatch latch) { + this.latch = latch; + } +- ++ public void setEnqueueTime(long enqueueTime) { ++ this.enqueueTime = enqueueTime; ++ } + public void idempotentRelease() { + idempotentRelease(true); + } +diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java +index b7392cc45..3c7b9b67f 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java ++++ b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java +@@ -16,6 +16,9 @@ + */ + package org.apache.rocketmq.store.timer; + ++import org.apache.rocketmq.common.message.MessageAccessor; ++import org.apache.rocketmq.common.message.MessageConst; ++import org.apache.rocketmq.common.message.MessageExt; + import org.junit.Assert; + import org.junit.Test; + +@@ -31,8 +34,11 @@ public class TimerMetricsTest { + + TimerMetrics first = new TimerMetrics(baseDir); + Assert.assertTrue(first.load()); +- first.addAndGet("AAA", 1000); +- first.addAndGet("BBB", 2000); ++ MessageExt msg = new MessageExt(); ++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "AAA"); ++ first.addAndGet(msg, 1000); ++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "BBB"); ++ first.addAndGet(msg, 2000); + Assert.assertEquals(1000, first.getTimingCount("AAA")); + Assert.assertEquals(2000, first.getTimingCount("BBB")); + long curr = System.currentTimeMillis(); +-- +2.32.0.windows.2 + + +From d7e5c4d1a4e048cd97f0b29a96a0fc575927a03e Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Mon, 25 Sep 2023 13:37:36 +0800 +Subject: [PATCH 06/12] [ISSUE #7389] Fix the problem that getLastMappedFile + function affects performance + +Co-authored-by: guyinyou +--- + .../apache/rocketmq/store/MappedFileQueue.java | 15 +++++++++++++-- + 1 file changed, 13 insertions(+), 2 deletions(-) + +diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +index 32b90d14f..9a0824829 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +@@ -377,8 +377,19 @@ public class MappedFileQueue implements Swappable { + } + + public MappedFile getLastMappedFile() { +- MappedFile[] mappedFiles = this.mappedFiles.toArray(new MappedFile[0]); +- return mappedFiles.length == 0 ? null : mappedFiles[mappedFiles.length - 1]; ++ MappedFile mappedFileLast = null; ++ while (!this.mappedFiles.isEmpty()) { ++ try { ++ mappedFileLast = this.mappedFiles.get(this.mappedFiles.size() - 1); ++ break; ++ } catch (IndexOutOfBoundsException e) { ++ //continue; ++ } catch (Exception e) { ++ log.error("getLastMappedFile has exception.", e); ++ break; ++ } ++ } ++ return mappedFileLast; + } + + public boolean resetOffset(long offset) { +-- +2.32.0.windows.2 + + +From 3fd43353fdf880deb5d63ba3ad50cc6e3259dc3a Mon Sep 17 00:00:00 2001 +From: Zhouxiang Zhan +Date: Tue, 26 Sep 2023 13:53:51 +0800 +Subject: [PATCH 07/12] [ISSUE #7393] Add timeout configuration for grpc server + (#7394) + +* Add timeout configuration for grpc server + +* Add proxyConfig +--- + .../java/org/apache/rocketmq/proxy/ProxyStartup.java | 1 + + .../apache/rocketmq/proxy/config/ProxyConfig.java | 9 +++++++++ + .../org/apache/rocketmq/proxy/grpc/GrpcServer.java | 10 ++++++++-- + .../rocketmq/proxy/grpc/GrpcServerBuilder.java | 12 +++++++++++- + 4 files changed, 29 insertions(+), 3 deletions(-) + +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java +index 06d5f4525..3b2ca99bf 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java +@@ -85,6 +85,7 @@ public class ProxyStartup { + .addService(ChannelzService.newInstance(100)) + .addService(ProtoReflectionService.newInstance()) + .configInterceptor(accessValidators) ++ .shutdownTime(ConfigurationManager.getProxyConfig().getGrpcShutdownTimeSeconds(), TimeUnit.SECONDS) + .build(); + PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer); + +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +index b2478fec3..c0d00d864 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +@@ -87,6 +87,7 @@ public class ProxyConfig implements ConfigFile { + */ + private String proxyMode = ProxyMode.CLUSTER.name(); + private Integer grpcServerPort = 8081; ++ private long grpcShutdownTimeSeconds = 30; + private int grpcBossLoopNum = 1; + private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2; + private boolean enableGrpcEpoll = false; +@@ -443,6 +444,14 @@ public class ProxyConfig implements ConfigFile { + this.grpcServerPort = grpcServerPort; + } + ++ public long getGrpcShutdownTimeSeconds() { ++ return grpcShutdownTimeSeconds; ++ } ++ ++ public void setGrpcShutdownTimeSeconds(long grpcShutdownTimeSeconds) { ++ this.grpcShutdownTimeSeconds = grpcShutdownTimeSeconds; ++ } ++ + public boolean isUseEndpointPortFromRequest() { + return useEndpointPortFromRequest; + } +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java +index 1bffa3c0b..d5b896fe1 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java +@@ -29,8 +29,14 @@ public class GrpcServer implements StartAndShutdown { + + private final Server server; + +- protected GrpcServer(Server server) { ++ private final long timeout; ++ ++ private final TimeUnit unit; ++ ++ protected GrpcServer(Server server, long timeout, TimeUnit unit) { + this.server = server; ++ this.timeout = timeout; ++ this.unit = unit; + } + + public void start() throws Exception { +@@ -40,7 +46,7 @@ public class GrpcServer implements StartAndShutdown { + + public void shutdown() { + try { +- this.server.shutdown().awaitTermination(30, TimeUnit.SECONDS); ++ this.server.shutdown().awaitTermination(timeout, unit); + log.info("grpc server shutdown successfully."); + } catch (Exception e) { + e.printStackTrace(); +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java +index 9cddd3013..0e79006f6 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java +@@ -41,6 +41,10 @@ public class GrpcServerBuilder { + private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + protected NettyServerBuilder serverBuilder; + ++ protected long time = 30; ++ ++ protected TimeUnit unit = TimeUnit.SECONDS; ++ + public static GrpcServerBuilder newBuilder(ThreadPoolExecutor executor, int port) { + return new GrpcServerBuilder(executor, port); + } +@@ -77,6 +81,12 @@ public class GrpcServerBuilder { + port, bossLoopNum, workerLoopNum, maxInboundMessageSize); + } + ++ public GrpcServerBuilder shutdownTime(long time, TimeUnit unit) { ++ this.time = time; ++ this.unit = unit; ++ return this; ++ } ++ + public GrpcServerBuilder addService(BindableService service) { + this.serverBuilder.addService(service); + return this; +@@ -93,7 +103,7 @@ public class GrpcServerBuilder { + } + + public GrpcServer build() { +- return new GrpcServer(this.serverBuilder.build()); ++ return new GrpcServer(this.serverBuilder.build(), time, unit); + } + + public GrpcServerBuilder configInterceptor(List accessValidators) { +-- +2.32.0.windows.2 + + +From c3b86cd1e3c068bc5847671c899a284e49a2ecdc Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= + +Date: Tue, 26 Sep 2023 16:07:13 +0800 +Subject: [PATCH 08/12] [ISSUE #7398] Fix ExportConfigsCommand NPE (#7399) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Co-authored-by: 石臻臻 +--- + .../command/export/ExportConfigsCommand.java | 42 ++++++++++++------- + 1 file changed, 26 insertions(+), 16 deletions(-) + +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java +index 03613b29c..c3f96d597 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java +@@ -20,6 +20,7 @@ import java.util.HashMap; + import java.util.List; + import java.util.Map; + import java.util.Map.Entry; ++import java.util.Arrays; + import java.util.Properties; + + import com.alibaba.fastjson.JSON; +@@ -106,24 +107,33 @@ public class ExportConfigsCommand implements SubCommand { + } + } + ++ + private Properties needBrokerProprties(Properties properties) { ++ List propertyKeys = Arrays.asList( ++ "brokerClusterName", ++ "brokerId", ++ "brokerName", ++ "brokerRole", ++ "fileReservedTime", ++ "filterServerNums", ++ "flushDiskType", ++ "maxMessageSize", ++ "messageDelayLevel", ++ "msgTraceTopicName", ++ "slaveReadEnable", ++ "traceOn", ++ "traceTopicEnable", ++ "useTLS", ++ "autoCreateTopicEnable", ++ "autoCreateSubscriptionGroup" ++ ); ++ + Properties newProperties = new Properties(); +- newProperties.setProperty("brokerClusterName", properties.getProperty("brokerClusterName")); +- newProperties.setProperty("brokerId", properties.getProperty("brokerId")); +- newProperties.setProperty("brokerName", properties.getProperty("brokerName")); +- newProperties.setProperty("brokerRole", properties.getProperty("brokerRole")); +- newProperties.setProperty("fileReservedTime", properties.getProperty("fileReservedTime")); +- newProperties.setProperty("filterServerNums", properties.getProperty("filterServerNums")); +- newProperties.setProperty("flushDiskType", properties.getProperty("flushDiskType")); +- newProperties.setProperty("maxMessageSize", properties.getProperty("maxMessageSize")); +- newProperties.setProperty("messageDelayLevel", properties.getProperty("messageDelayLevel")); +- newProperties.setProperty("msgTraceTopicName", properties.getProperty("msgTraceTopicName")); +- newProperties.setProperty("slaveReadEnable", properties.getProperty("slaveReadEnable")); +- newProperties.setProperty("traceOn", properties.getProperty("traceOn")); +- newProperties.setProperty("traceTopicEnable", properties.getProperty("traceTopicEnable")); +- newProperties.setProperty("useTLS", properties.getProperty("useTLS")); +- newProperties.setProperty("autoCreateTopicEnable", properties.getProperty("autoCreateTopicEnable")); +- newProperties.setProperty("autoCreateSubscriptionGroup", properties.getProperty("autoCreateSubscriptionGroup")); ++ propertyKeys.stream() ++ .filter(key -> properties.getProperty(key) != null) ++ .forEach(key -> newProperties.setProperty(key, properties.getProperty(key))); ++ + return newProperties; + } ++ + } +-- +2.32.0.windows.2 + + +From 959a98120cea8022498557a308aff35e3d8def2a Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= + +Date: Wed, 27 Sep 2023 01:59:58 +0800 +Subject: [PATCH 09/12] [ISSUE #7400] Fix getBrokerEpochSubCommand NPE +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Co-authored-by: 石臻臻 +--- + .../broker/processor/AdminBrokerProcessor.java | 10 ++++++++-- + 1 file changed, 8 insertions(+), 2 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +index 9e48431be..e77120e15 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +@@ -2736,10 +2736,16 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + final ReplicasManager replicasManager = this.brokerController.getReplicasManager(); + assert replicasManager != null; + final BrokerConfig brokerConfig = this.brokerController.getBrokerConfig(); ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ ++ if (!brokerConfig.isEnableControllerMode()) { ++ response.setCode(ResponseCode.SYSTEM_ERROR); ++ response.setRemark("this request only for controllerMode "); ++ return response; ++ } + final EpochEntryCache entryCache = new EpochEntryCache(brokerConfig.getBrokerClusterName(), +- brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset()); ++ brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset()); + +- final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setBody(entryCache.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); +-- +2.32.0.windows.2 + + +From 0a6ae4605fef4eaab6262fbd370aba887d8ae58b Mon Sep 17 00:00:00 2001 +From: tiger lee <1026203200@qq.com> +Date: Wed, 27 Sep 2023 14:43:15 +0800 +Subject: [PATCH 10/12] [ISSUE #7396] Fix wrong word in + BrokerController#doResterBrokerAll (#7397) + +--- + .../main/java/org/apache/rocketmq/broker/BrokerController.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index 53e2e1b62..d4bded600 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -1807,7 +1807,7 @@ public class BrokerController { + TopicConfigSerializeWrapper topicConfigWrapper) { + + if (shutdown) { +- BrokerController.LOG.info("BrokerController#doResterBrokerAll: broker has shutdown, no need to register any more."); ++ BrokerController.LOG.info("BrokerController#doRegisterBrokerAll: broker has shutdown, no need to register any more."); + return; + } + List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( +-- +2.32.0.windows.2 + + +From 4f1b42a7c5557bcadd6b9982a0c9bd876622c69e Mon Sep 17 00:00:00 2001 +From: ShuangxiDing +Date: Thu, 28 Sep 2023 16:52:02 +0800 +Subject: [PATCH 11/12] [ISSUE #7410] Handle the Exception when the Proxy + requests the client +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Co-authored-by: 徒钟 +--- + .../remoting/channel/RemotingChannel.java | 23 ++++++++++++++----- + 1 file changed, 17 insertions(+), 6 deletions(-) + +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java +index 40946cabf..d755fdcc4 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java +@@ -34,6 +34,7 @@ import org.apache.rocketmq.common.utils.NetworkUtil; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.proxy.common.channel.ChannelHelper; ++import org.apache.rocketmq.proxy.common.utils.ExceptionUtils; + import org.apache.rocketmq.proxy.common.utils.FutureUtils; + import org.apache.rocketmq.proxy.config.ConfigurationManager; + import org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter; +@@ -158,10 +159,15 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver + if (response.getCode() == ResponseCode.SUCCESS) { + ConsumerRunningInfo consumerRunningInfo = ConsumerRunningInfo.decode(response.getBody(), ConsumerRunningInfo.class); + responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", consumerRunningInfo)); ++ } else { ++ String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark()); ++ RuntimeException e = new RuntimeException(errMsg); ++ responseFuture.completeExceptionally(e); + } +- String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark()); +- RuntimeException e = new RuntimeException(errMsg); +- responseFuture.completeExceptionally(e); ++ }) ++ .exceptionally(t -> { ++ responseFuture.completeExceptionally(ExceptionUtils.getRealException(t)); ++ return null; + }); + return CompletableFuture.completedFuture(null); + } catch (Throwable t) { +@@ -183,10 +189,15 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver + if (response.getCode() == ResponseCode.SUCCESS) { + ConsumeMessageDirectlyResult result = ConsumeMessageDirectlyResult.decode(response.getBody(), ConsumeMessageDirectlyResult.class); + responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", result)); ++ } else { ++ String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark()); ++ RuntimeException e = new RuntimeException(errMsg); ++ responseFuture.completeExceptionally(e); + } +- String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark()); +- RuntimeException e = new RuntimeException(errMsg); +- responseFuture.completeExceptionally(e); ++ }) ++ .exceptionally(t -> { ++ responseFuture.completeExceptionally(ExceptionUtils.getRealException(t)); ++ return null; + }); + return CompletableFuture.completedFuture(null); + } catch (Throwable t) { +-- +2.32.0.windows.2 + + +From c36bb78e850129b9db40adc5b0e1b9bfd5c8fd2e Mon Sep 17 00:00:00 2001 +From: shriVATSA54 <116296557+shriVATSA54@users.noreply.github.com> +Date: Sat, 7 Oct 2023 12:22:39 +0530 +Subject: [PATCH 12/12] [ISSUE 7313] Enhancement Optimization Method name + (#7420) + +* Enhancment/method_name/#7313/ + +* Enhancment/method_name/#7313/ + +* Enhancment/method_name/#7313/ +--- + .../tieredstore/provider/TieredStoreTopicBlackListFilter.java | 2 +- + .../rocketmq/tieredstore/provider/TieredStoreTopicFilter.java | 2 +- + .../provider/TieredStoreTopicBlackListFilterTest.java | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java +index 50adbb713..f8bf165bc 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java +@@ -39,7 +39,7 @@ public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter { + } + + @Override +- public void addTopicToWhiteList(String topicName) { ++ public void addTopicToBlackList(String topicName) { + this.topicBlackSet.add(topicName); + } + } +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java +index 3f26b8b02..f983ed6e9 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java +@@ -21,5 +21,5 @@ public interface TieredStoreTopicFilter { + + boolean filterTopic(String topicName); + +- void addTopicToWhiteList(String topicName); ++ void addTopicToBlackList(String topicName); + } +diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java +index 2bf48173c..fbaafa1b4 100644 +--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java +@@ -30,7 +30,7 @@ public class TieredStoreTopicBlackListFilterTest { + + String topicName = "WhiteTopic"; + Assert.assertFalse(topicFilter.filterTopic(topicName)); +- topicFilter.addTopicToWhiteList(topicName); ++ topicFilter.addTopicToBlackList(topicName); + Assert.assertTrue(topicFilter.filterTopic(topicName)); + } + } +\ No newline at end of file +-- +2.32.0.windows.2 + diff --git a/patch020-backport-add-goaway-mechanism.patch b/patch020-backport-add-goaway-mechanism.patch new file mode 100644 index 000000000..122db0325 --- /dev/null +++ b/patch020-backport-add-goaway-mechanism.patch @@ -0,0 +1,3696 @@ +From 84156084a4c5228e1d2fe21e068fff330bbc40d1 Mon Sep 17 00:00:00 2001 +From: Zhouxiang Zhan +Date: Sun, 8 Oct 2023 11:13:25 +0800 +Subject: [PATCH 1/7] [ISSUE #7321] Refector NettyRemotingAbstract with unify + future implementation (#7322) + +* Refector NettyRemotingAbstract + +* Add invoke with future method + +* Deprecate InvokeCallback#operationComplete + +* Add operationSuccess and operationException for InvokeCallback + +* fix unit test + +* fix unit test + +* Keep InvokeCallback#operationComplete + +* Optimize invokeAsyncImpl operationComplete + +* Add unit test for NettyRemotingClient + +* fix checkstyle +--- + .../rocketmq/broker/out/BrokerOuterAPI.java | 147 +++++---- + .../rocketmq/client/impl/MQAdminImpl.java | 71 ++-- + .../rocketmq/client/impl/MQClientAPIImpl.java | 239 +++++++------- + .../client/impl/mqclient/MQClientAPIExt.java | 309 ++++++++---------- + .../client/impl/MQClientAPIImplTest.java | 12 +- + .../remoting/RemotingProtocolServer.java | 22 +- + .../service/mqclient/MQClientAPIExtTest.java | 97 +++--- + .../rocketmq/remoting/InvokeCallback.java | 15 + + .../rocketmq/remoting/RemotingClient.java | 27 +- + .../remoting/netty/NettyRemotingAbstract.java | 123 ++++--- + .../remoting/netty/NettyRemotingClient.java | 33 +- + .../remoting/netty/ResponseFuture.java | 15 + + .../rocketmq/remoting/rpc/RpcClientImpl.java | 29 +- + .../rocketmq/remoting/RemotingServerTest.java | 22 +- + .../rocketmq/remoting/netty/MockChannel.java | 21 +- + .../remoting/netty/MockChannelPromise.java | 191 +++++++++++ + .../netty/NettyRemotingAbstractTest.java | 54 ++- + .../netty/NettyRemotingClientTest.java | 185 ++++++++++- + 18 files changed, 1029 insertions(+), 583 deletions(-) + rename client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java => remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java (57%) + create mode 100644 remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +index 9dfb8127d..6fde48dd9 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +@@ -73,6 +73,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; + import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; + import org.apache.rocketmq.remoting.netty.NettyClientConfig; + import org.apache.rocketmq.remoting.netty.NettyRemotingClient; ++import org.apache.rocketmq.remoting.netty.ResponseFuture; + import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo; + import org.apache.rocketmq.remoting.protocol.DataVersion; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; +@@ -107,6 +108,8 @@ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; + import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader; ++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; ++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader; +@@ -124,8 +127,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerReques + import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader; + import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterTopicRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader; +-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; + import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; + import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult; + import org.apache.rocketmq.remoting.protocol.route.BrokerData; +@@ -151,7 +152,6 @@ public class BrokerOuterAPI { + private final RpcClient rpcClient; + private String nameSrvAddr = null; + +- + public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { + this(nettyClientConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata()); + } +@@ -459,7 +459,7 @@ public class BrokerOuterAPI { + * @param filterServerList + * @param oneway + * @param timeoutMills +- * @param compressed default false ++ * @param compressed default false + * @return + */ + public List registerBrokerAll( +@@ -643,7 +643,6 @@ public class BrokerOuterAPI { + queueDatas.add(queueData); + final byte[] topicRouteBody = topicRouteData.encode(); + +- + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size()); + for (final String namesrvAddr : nameServerAddressList) { +@@ -910,25 +909,33 @@ public class BrokerOuterAPI { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); + + request.setBody(requestBody.encode()); +- this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { +- if (callback == null) { +- return; ++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ + } + +- try { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- if (response.getCode() == ResponseCode.SUCCESS) { +- LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), +- LockBatchResponseBody.class); +- Set messageQueues = responseBody.getLockOKMQSet(); +- callback.onSuccess(messageQueues); +- } else { +- callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); +- } ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ if (callback == null) { ++ return; + } +- } catch (Throwable ignored) { ++ if (response.getCode() == ResponseCode.SUCCESS) { ++ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), ++ LockBatchResponseBody.class); ++ Set messageQueues = responseBody.getLockOKMQSet(); ++ callback.onSuccess(messageQueues); ++ } else { ++ callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); ++ } ++ } + ++ @Override ++ public void operationFail(Throwable throwable) { ++ if (callback == null) { ++ return; ++ } ++ callback.onException(throwable); + } + }); + } +@@ -942,22 +949,30 @@ public class BrokerOuterAPI { + + request.setBody(requestBody.encode()); + +- this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { +- if (callback == null) { +- return; ++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ + } + +- try { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- if (response.getCode() == ResponseCode.SUCCESS) { +- callback.onSuccess(); +- } else { +- callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); +- } ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ if (callback == null) { ++ return; + } +- } catch (Throwable ignored) { ++ if (response.getCode() == ResponseCode.SUCCESS) { ++ callback.onSuccess(); ++ } else { ++ callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); ++ } ++ } + ++ @Override ++ public void operationFail(Throwable throwable) { ++ if (callback == null) { ++ return; ++ } ++ callback.onException(throwable); + } + }); + } +@@ -983,21 +998,27 @@ public class BrokerOuterAPI { + CompletableFuture cf = new CompletableFuture<>(); + final String msgId = msg.getMsgId(); + try { +- this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (null != response) { +- SendResult sendResult = null; ++ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + try { +- sendResult = this.processSendResponse(brokerName, msg, response); ++ SendResult sendResult = processSendResponse(brokerName, msg, response); + cf.complete(sendResult); + } catch (MQBrokerException | RemotingCommandException e) { + LOGGER.error("processSendResponse in sendMessageToSpecificBrokerAsync failed, msgId=" + msgId, e); + cf.completeExceptionally(e); + } +- } else { +- cf.complete(null); + } + ++ @Override ++ public void operationFail(Throwable throwable) { ++ cf.completeExceptionally(throwable); ++ } + }); + } catch (Throwable t) { + LOGGER.error("invokeAsync failed in sendMessageToSpecificBrokerAsync, msgId=" + msgId, t); +@@ -1057,7 +1078,7 @@ public class BrokerOuterAPI { + } + if (sendStatus != null) { + SendMessageResponseHeader responseHeader = +- (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); ++ (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); + + //If namespace not null , reset Topic without namespace. + String topic = msg.getTopic(); +@@ -1073,8 +1094,8 @@ public class BrokerOuterAPI { + uniqMsgId = sb.toString(); + } + SendResult sendResult = new SendResult(sendStatus, +- uniqMsgId, +- responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); ++ uniqMsgId, ++ responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); + sendResult.setTransactionId(responseHeader.getTransactionId()); + String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); + String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH); +@@ -1218,8 +1239,9 @@ public class BrokerOuterAPI { + /** + * Broker try to elect itself as a master in broker set + */ +- public Pair> brokerElect(String controllerAddress, String clusterName, String brokerName, +- Long brokerId) throws Exception { ++ public Pair> brokerElect(String controllerAddress, String clusterName, ++ String brokerName, ++ Long brokerId) throws Exception { + + final ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, requestHeader); +@@ -1237,7 +1259,8 @@ public class BrokerOuterAPI { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + +- public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, final String controllerAddress) throws Exception { ++ public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, ++ final String controllerAddress) throws Exception { + final GetNextBrokerIdRequestHeader requestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName); + final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, requestHeader); + final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); +@@ -1248,7 +1271,8 @@ public class BrokerOuterAPI { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + +- public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { ++ public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, ++ final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { + final ApplyBrokerIdRequestHeader requestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, brokerId, registerCheckCode); + final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, requestHeader); + final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); +@@ -1259,7 +1283,9 @@ public class BrokerOuterAPI { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + +- public Pair> registerBrokerToController(final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, final String controllerAddress) throws Exception { ++ public Pair> registerBrokerToController( ++ final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, ++ final String controllerAddress) throws Exception { + final RegisterBrokerToControllerRequestHeader requestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, brokerId, brokerAddress); + final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, requestHeader); + final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); +@@ -1355,16 +1381,25 @@ public class BrokerOuterAPI { + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader); + CompletableFuture pullResultFuture = new CompletableFuture<>(); +- this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- if (responseFuture.getCause() != null) { +- pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); +- return; ++ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ + } +- try { +- PullResultExt pullResultExt = this.processPullResponse(responseFuture.getResponseCommand(), brokerAddr); +- this.processPullResult(pullResultExt, brokerName, queueId); +- pullResultFuture.complete(pullResultExt); +- } catch (Exception e) { ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ try { ++ PullResultExt pullResultExt = processPullResponse(response, brokerAddr); ++ processPullResult(pullResultExt, brokerName, queueId); ++ pullResultFuture.complete(pullResultExt); ++ } catch (Exception e) { ++ pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); ++ } ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { + pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); + } + }); +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +index 1ef3a9483..83835bd3d 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +@@ -44,6 +44,8 @@ import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.message.MessageId; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.common.utils.NetworkUtil; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.remoting.InvokeCallback; + import org.apache.rocketmq.remoting.exception.RemotingCommandException; + import org.apache.rocketmq.remoting.exception.RemotingException; +@@ -55,8 +57,6 @@ import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; + import org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader; + import org.apache.rocketmq.remoting.protocol.route.BrokerData; + import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +-import org.apache.rocketmq.logging.org.slf4j.Logger; +-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + + public class MQAdminImpl { + +@@ -357,44 +357,51 @@ public class MQAdminImpl { + new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + try { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- switch (response.getCode()) { +- case ResponseCode.SUCCESS: { +- QueryMessageResponseHeader responseHeader = null; +- try { +- responseHeader = +- (QueryMessageResponseHeader) response +- .decodeCommandCustomHeader(QueryMessageResponseHeader.class); +- } catch (RemotingCommandException e) { +- log.error("decodeCommandCustomHeader exception", e); +- return; +- } +- +- List wrappers = +- MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); +- +- QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); +- try { +- lock.writeLock().lock(); +- queryResultList.add(qr); +- } finally { +- lock.writeLock().unlock(); +- } +- break; ++ switch (response.getCode()) { ++ case ResponseCode.SUCCESS: { ++ QueryMessageResponseHeader responseHeader = null; ++ try { ++ responseHeader = ++ (QueryMessageResponseHeader) response ++ .decodeCommandCustomHeader(QueryMessageResponseHeader.class); ++ } catch (RemotingCommandException e) { ++ log.error("decodeCommandCustomHeader exception", e); ++ return; + } +- default: +- log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); +- break; ++ ++ List wrappers = ++ MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); ++ ++ QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); ++ try { ++ lock.writeLock().lock(); ++ queryResultList.add(qr); ++ } finally { ++ lock.writeLock().unlock(); ++ } ++ break; + } +- } else { +- log.warn("getResponseCommand return null"); ++ default: ++ log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); ++ break; + } ++ + } finally { + countDownLatch.countDown(); + } + } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ log.error("queryMessage error, requestHeader={}", requestHeader); ++ countDownLatch.countDown(); ++ } + }, isUniqKey); + } catch (Exception e) { + log.warn("queryMessage exception", e); +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +index 3201a493f..2407e5737 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +@@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger; + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.client.ClientConfig; + import org.apache.rocketmq.client.Validators; +-import org.apache.rocketmq.client.common.ClientErrorCode; + import org.apache.rocketmq.client.consumer.AckCallback; + import org.apache.rocketmq.client.consumer.AckResult; + import org.apache.rocketmq.client.consumer.AckStatus; +@@ -653,10 +652,13 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { +- long cost = System.currentTimeMillis() - beginStartTime; +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (null == sendCallback && response != null) { + ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ long cost = System.currentTimeMillis() - beginStartTime; ++ if (null == sendCallback) { + try { + SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); + if (context != null && sendResult != null) { +@@ -666,46 +668,47 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + } catch (Throwable e) { + } + +- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); ++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); + return; + } + +- if (response != null) { ++ try { ++ SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); ++ assert sendResult != null; ++ if (context != null) { ++ context.setSendResult(sendResult); ++ context.getProducer().executeSendMessageHookAfter(context); ++ } ++ + try { +- SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); +- assert sendResult != null; +- if (context != null) { +- context.setSendResult(sendResult); +- context.getProducer().executeSendMessageHookAfter(context); +- } ++ sendCallback.onSuccess(sendResult); ++ } catch (Throwable e) { ++ } + +- try { +- sendCallback.onSuccess(sendResult); +- } catch (Throwable e) { +- } ++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); ++ } catch (Exception e) { ++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++ retryTimesWhenSendFailed, times, e, context, false, producer); ++ } ++ } + +- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); +- } catch (Exception e) { +- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); +- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +- retryTimesWhenSendFailed, times, e, context, false, producer); +- } ++ @Override ++ public void operationFail(Throwable throwable) { ++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); ++ long cost = System.currentTimeMillis() - beginStartTime; ++ if (throwable instanceof RemotingSendRequestException) { ++ MQClientException ex = new MQClientException("send request failed", throwable); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++ retryTimesWhenSendFailed, times, ex, context, true, producer); ++ } else if (throwable instanceof RemotingTimeoutException) { ++ MQClientException ex = new MQClientException("wait response timeout, cost=" + cost, throwable); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++ retryTimesWhenSendFailed, times, ex, context, true, producer); + } else { +- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); +- if (!responseFuture.isSendRequestOK()) { +- MQClientException ex = new MQClientException("send request failed", responseFuture.getCause()); +- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +- retryTimesWhenSendFailed, times, ex, context, true, producer); +- } else if (responseFuture.isTimeout()) { +- MQClientException ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", +- responseFuture.getCause()); +- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +- retryTimesWhenSendFailed, times, ex, context, true, producer); +- } else { +- MQClientException ex = new MQClientException("unknow reseaon", responseFuture.getCause()); +- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +- retryTimesWhenSendFailed, times, ex, context, true, producer); +- } ++ MQClientException ex = new MQClientException("unknow reseaon", throwable); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++ retryTimesWhenSendFailed, times, ex, context, true, producer); + } + } + }); +@@ -857,30 +860,25 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + final long timeoutMillis, final PopCallback popCallback + ) throws RemotingException, InterruptedException { + final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader); +- this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { ++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override +- public void onComplete(ResponseFuture responseFuture) { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- try { +- PopResult +- popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); +- assert popResult != null; +- popCallback.onSuccess(popResult); +- } catch (Exception e) { +- popCallback.onException(e); +- } +- } else { +- if (!responseFuture.isSendRequestOK()) { +- popCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); +- } else if (responseFuture.isTimeout()) { +- popCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, +- responseFuture.getCause())); +- } else { +- popCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); +- } ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ try { ++ PopResult popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); ++ popCallback.onSuccess(popResult); ++ } catch (Exception e) { ++ popCallback.onException(e); + } + } ++ @Override ++ public void operationFail(Throwable throwable) { ++ popCallback.onException(throwable); ++ } + }); + } + +@@ -959,34 +957,26 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + request.setBody(requestBody.encode()); + } + } +- this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) { ++ this.remotingClient.invokeAsync(addr, request, timeOut, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } + + @Override +- public void onComplete(ResponseFuture responseFuture) { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- try { +- AckResult ackResult = new AckResult(); +- if (ResponseCode.SUCCESS == response.getCode()) { +- ackResult.setStatus(AckStatus.OK); +- } else { +- ackResult.setStatus(AckStatus.NO_EXIST); +- } +- ackCallback.onSuccess(ackResult); +- } catch (Exception e) { +- ackCallback.onException(e); +- } ++ public void operationSucceed(RemotingCommand response) { ++ AckResult ackResult = new AckResult(); ++ if (ResponseCode.SUCCESS == response.getCode()) { ++ ackResult.setStatus(AckStatus.OK); + } else { +- if (!responseFuture.isSendRequestOK()) { +- ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); +- } else if (responseFuture.isTimeout()) { +- ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, +- responseFuture.getCause())); +- } else { +- ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeOut + ". Request: " + request, responseFuture.getCause())); +- } ++ ackResult.setStatus(AckStatus.NO_EXIST); + } ++ ackCallback.onSuccess(ackResult); ++ } + ++ @Override ++ public void operationFail(Throwable throwable) { ++ ackCallback.onException(throwable); + } + }); + } +@@ -999,39 +989,37 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + final AckCallback ackCallback + ) throws RemotingException, MQBrokerException, InterruptedException { + final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader); +- this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { ++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override +- public void onComplete(ResponseFuture responseFuture) { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- try { +- ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); +- AckResult ackResult = new AckResult(); +- if (ResponseCode.SUCCESS == response.getCode()) { +- ackResult.setStatus(AckStatus.OK); +- ackResult.setPopTime(responseHeader.getPopTime()); +- ackResult.setExtraInfo(ExtraInfoUtil +- .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), +- responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR +- + requestHeader.getOffset()); +- } else { +- ackResult.setStatus(AckStatus.NO_EXIST); +- } +- ackCallback.onSuccess(ackResult); +- } catch (Exception e) { +- ackCallback.onException(e); +- } +- } else { +- if (!responseFuture.isSendRequestOK()) { +- ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); +- } else if (responseFuture.isTimeout()) { +- ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, +- responseFuture.getCause())); ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ try { ++ ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); ++ AckResult ackResult = new AckResult(); ++ if (ResponseCode.SUCCESS == response.getCode()) { ++ ackResult.setStatus(AckStatus.OK); ++ ackResult.setPopTime(responseHeader.getPopTime()); ++ ackResult.setExtraInfo(ExtraInfoUtil ++ .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), ++ responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR ++ + requestHeader.getOffset()); + } else { +- ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); ++ ackResult.setStatus(AckStatus.NO_EXIST); + } ++ ackCallback.onSuccess(ackResult); ++ } catch (Exception e) { ++ ackCallback.onException(e); + } + } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ ackCallback.onException(throwable); ++ } + }); + } + +@@ -1044,26 +1032,23 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- try { +- PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); +- assert pullResult != null; +- pullCallback.onSuccess(pullResult); +- } catch (Exception e) { +- pullCallback.onException(e); +- } +- } else { +- if (!responseFuture.isSendRequestOK()) { +- pullCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); +- } else if (responseFuture.isTimeout()) { +- pullCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, +- responseFuture.getCause())); +- } else { +- pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); +- } ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ try { ++ PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); ++ pullCallback.onSuccess(pullResult); ++ } catch (Exception e) { ++ pullCallback.onException(e); + } + } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ pullCallback.onException(throwable); ++ } + }); + } + +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +index d7c8ef8d9..f3102e175 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +@@ -30,7 +30,6 @@ import org.apache.rocketmq.client.consumer.PullCallback; + import org.apache.rocketmq.client.consumer.PullResult; + import org.apache.rocketmq.client.consumer.PullStatus; + import org.apache.rocketmq.client.exception.MQBrokerException; +-import org.apache.rocketmq.client.exception.MQClientException; + import org.apache.rocketmq.client.exception.OffsetNotFoundException; + import org.apache.rocketmq.client.impl.ClientRemotingProcessor; + import org.apache.rocketmq.client.impl.CommunicationMode; +@@ -47,6 +46,7 @@ import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.remoting.InvokeCallback; + import org.apache.rocketmq.remoting.RPCHook; + import org.apache.rocketmq.remoting.exception.RemotingCommandException; + import org.apache.rocketmq.remoting.netty.NettyClientConfig; +@@ -106,19 +106,6 @@ public class MQClientAPIExt extends MQClientAPIImpl { + return false; + } + +- protected static MQClientException processNullResponseErr(ResponseFuture responseFuture) { +- MQClientException ex; +- if (!responseFuture.isSendRequestOK()) { +- ex = new MQClientException("send request failed", responseFuture.getCause()); +- } else if (responseFuture.isTimeout()) { +- ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", +- responseFuture.getCause()); +- } else { +- ex = new MQClientException("unknown reason", responseFuture.getCause()); +- } +- return ex; +- } +- + public CompletableFuture sendHeartbeatOneway( + String brokerAddr, + HeartbeatData heartbeatData, +@@ -146,24 +133,15 @@ public class MQClientAPIExt extends MQClientAPIImpl { + request.setLanguage(clientConfig.getLanguage()); + request.setBody(heartbeatData.encode()); + +- CompletableFuture future = new CompletableFuture<>(); +- try { +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- if (ResponseCode.SUCCESS == response.getCode()) { +- future.complete(response.getVersion()); +- } else { +- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); +- } +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); +- } +- }); +- } catch (Throwable t) { +- future.completeExceptionally(t); +- } +- return future; ++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { ++ CompletableFuture future0 = new CompletableFuture<>(); ++ if (ResponseCode.SUCCESS == response.getCode()) { ++ future0.complete(response.getVersion()); ++ } else { ++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); ++ } ++ return future0; ++ }); + } + + public CompletableFuture sendMessageAsync( +@@ -177,24 +155,15 @@ public class MQClientAPIExt extends MQClientAPIImpl { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2); + request.setBody(msg.getBody()); + +- CompletableFuture future = new CompletableFuture<>(); +- try { +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- try { +- future.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); +- } catch (Exception e) { +- future.completeExceptionally(e); +- } +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); +- } +- }); +- } catch (Throwable t) { +- future.completeExceptionally(t); +- } +- return future; ++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { ++ CompletableFuture future0 = new CompletableFuture<>(); ++ try { ++ future0.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); ++ } catch (Exception e) { ++ future0.completeExceptionally(e); ++ } ++ return future0; ++ }); + } + + public CompletableFuture sendMessageAsync( +@@ -216,17 +185,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { + msgBatch.setBody(body); + + request.setBody(body); +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- try { +- future.complete(this.processSendResponse(brokerName, msgBatch, response, brokerAddr)); +- } catch (Exception e) { +- future.completeExceptionally(e); +- } +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); ++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { ++ CompletableFuture future0 = new CompletableFuture<>(); ++ try { ++ future0.complete(processSendResponse(brokerName, msgBatch, response, brokerAddr)); ++ } catch (Exception e) { ++ future0.completeExceptionally(e); + } ++ return future0; + }); + } catch (Throwable t) { + future.completeExceptionally(t); +@@ -240,21 +206,7 @@ public class MQClientAPIExt extends MQClientAPIImpl { + long timeoutMillis + ) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); +- +- CompletableFuture future = new CompletableFuture<>(); +- try { +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- future.complete(response); +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); +- } +- }); +- } catch (Throwable t) { +- future.completeExceptionally(t); +- } +- return future; ++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis); + } + + public CompletableFuture popMessageAsync( +@@ -402,38 +354,31 @@ public class MQClientAPIExt extends MQClientAPIImpl { + QueryConsumerOffsetRequestHeader requestHeader, + long timeoutMillis + ) { +- CompletableFuture future = new CompletableFuture<>(); +- try { +- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- switch (response.getCode()) { +- case ResponseCode.SUCCESS: { +- try { +- QueryConsumerOffsetResponseHeader responseHeader = +- (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); +- future.complete(responseHeader.getOffset()); +- } catch (RemotingCommandException e) { +- future.completeExceptionally(e); +- } +- break; +- } +- case ResponseCode.QUERY_NOT_FOUND: { +- future.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); +- break; +- } +- default: +- break; ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); ++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { ++ CompletableFuture future0 = new CompletableFuture<>(); ++ switch (response.getCode()) { ++ case ResponseCode.SUCCESS: { ++ try { ++ QueryConsumerOffsetResponseHeader responseHeader = ++ (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); ++ future0.complete(responseHeader.getOffset()); ++ } catch (RemotingCommandException e) { ++ future0.completeExceptionally(e); + } +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); ++ break; + } +- }); +- } catch (Throwable t) { +- future.completeExceptionally(t); +- } +- return future; ++ case ResponseCode.QUERY_NOT_FOUND: { ++ future0.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); ++ break; ++ } ++ default: { ++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++ break; ++ } ++ } ++ return future0; ++ }); + } + + public CompletableFuture updateConsumerOffsetOneWay( +@@ -461,9 +406,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { + + CompletableFuture> future = new CompletableFuture<>(); + try { +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { ++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { +@@ -485,8 +435,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { + break; + } + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ future.completeExceptionally(throwable); + } + }); + } catch (Throwable t) { +@@ -501,9 +454,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { + + CompletableFuture future = new CompletableFuture<>(); + try { +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { ++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + if (ResponseCode.SUCCESS == response.getCode()) { + try { + GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); +@@ -513,8 +471,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { + } + } + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ future.completeExceptionally(throwable); + } + }); + } catch (Throwable t) { +@@ -529,9 +490,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { + + CompletableFuture future = new CompletableFuture<>(); + try { +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { ++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + if (ResponseCode.SUCCESS == response.getCode()) { + try { + GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); +@@ -541,8 +507,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { + } + } + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ future.completeExceptionally(throwable); + } + }); + } catch (Throwable t) { +@@ -555,57 +524,41 @@ public class MQClientAPIExt extends MQClientAPIImpl { + long timeoutMillis) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader); + +- CompletableFuture future = new CompletableFuture<>(); +- try { +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- if (response.getCode() == ResponseCode.SUCCESS) { +- try { +- SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); +- future.complete(responseHeader.getOffset()); +- } catch (Throwable t) { +- future.completeExceptionally(t); +- } +- } +- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); ++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { ++ CompletableFuture future0 = new CompletableFuture<>(); ++ if (response.getCode() == ResponseCode.SUCCESS) { ++ try { ++ SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); ++ future0.complete(responseHeader.getOffset()); ++ } catch (Throwable t) { ++ future0.completeExceptionally(t); + } +- }); +- } catch (Throwable t) { +- future.completeExceptionally(t); +- } +- return future; ++ } else { ++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++ } ++ return future0; ++ }); + } + + public CompletableFuture> lockBatchMQWithFuture(String brokerAddr, + LockBatchRequestBody requestBody, long timeoutMillis) { +- CompletableFuture> future = new CompletableFuture<>(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); + request.setBody(requestBody.encode()); +- try { +- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { +- if (response.getCode() == ResponseCode.SUCCESS) { +- try { +- LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); +- Set messageQueues = responseBody.getLockOKMQSet(); +- future.complete(messageQueues); +- } catch (Throwable t) { +- future.completeExceptionally(t); +- } +- } +- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +- } else { +- future.completeExceptionally(processNullResponseErr(responseFuture)); ++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { ++ CompletableFuture> future0 = new CompletableFuture<>(); ++ if (response.getCode() == ResponseCode.SUCCESS) { ++ try { ++ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); ++ Set messageQueues = responseBody.getLockOKMQSet(); ++ future0.complete(messageQueues); ++ } catch (Throwable t) { ++ future0.completeExceptionally(t); + } +- }); +- } catch (Exception e) { +- future.completeExceptionally(e); +- } +- return future; ++ } else { ++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++ } ++ return future0; ++ }); + } + + public CompletableFuture unlockBatchMQOneway(String brokerAddr, +@@ -624,25 +577,21 @@ public class MQClientAPIExt extends MQClientAPIImpl { + + public CompletableFuture notification(String brokerAddr, NotificationRequestHeader requestHeader, + long timeoutMillis) { +- CompletableFuture future = new CompletableFuture<>(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFICATION, requestHeader); +- try { +- this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenAccept(response -> { +- if (response.getCode() == ResponseCode.SUCCESS) { +- try { +- NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); +- future.complete(responseHeader.isHasMsg()); +- } catch (Throwable t) { +- future.completeExceptionally(t); +- } +- } else { +- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { ++ CompletableFuture future0 = new CompletableFuture<>(); ++ if (response.getCode() == ResponseCode.SUCCESS) { ++ try { ++ NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); ++ future0.complete(responseHeader.isHasMsg()); ++ } catch (Throwable t) { ++ future0.completeExceptionally(t); + } +- }); +- } catch (Throwable t) { +- future.completeExceptionally(t); +- } +- return future; ++ } else { ++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++ } ++ return future0; ++ }); + } + + public CompletableFuture invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) { +diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +index d13f2cfe4..c152d38ea 100644 +--- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java ++++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +@@ -212,7 +212,7 @@ public class MQClientAPIImplTest { + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); +- callback.operationComplete(responseFuture); ++ callback.operationSucceed(responseFuture.getResponseCommand()); + return null; + } + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +@@ -386,7 +386,7 @@ public class MQClientAPIImplTest { + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); +- callback.operationComplete(responseFuture); ++ callback.operationSucceed(responseFuture.getResponseCommand()); + return null; + } + }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class)); +@@ -472,7 +472,7 @@ public class MQClientAPIImplTest { + message.putUserProperty("key", "value"); + response.setBody(MessageDecoder.encode(message, false)); + responseFuture.setResponseCommand(response); +- callback.operationComplete(responseFuture); ++ callback.operationSucceed(responseFuture.getResponseCommand()); + return null; + } + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +@@ -543,7 +543,7 @@ public class MQClientAPIImplTest { + message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0)); + response.setBody(MessageDecoder.encode(message, false)); + responseFuture.setResponseCommand(response); +- callback.operationComplete(responseFuture); ++ callback.operationSucceed(responseFuture.getResponseCommand()); + return null; + } + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +@@ -585,7 +585,7 @@ public class MQClientAPIImplTest { + response.setOpaque(request.getOpaque()); + response.setCode(ResponseCode.SUCCESS); + responseFuture.setResponseCommand(response); +- callback.operationComplete(responseFuture); ++ callback.operationSucceed(responseFuture.getResponseCommand()); + return null; + } + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +@@ -622,7 +622,7 @@ public class MQClientAPIImplTest { + responseHeader.setPopTime(System.currentTimeMillis()); + responseHeader.setInvisibleTime(10 * 1000L); + responseFuture.setResponseCommand(response); +- callback.operationComplete(responseFuture); ++ callback.operationSucceed(responseFuture.getResponseCommand()); + return null; + } + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +index fe07090d5..3227d1e1c 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +@@ -26,7 +26,6 @@ import java.util.concurrent.ScheduledExecutorService; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.acl.AccessValidator; +-import org.apache.rocketmq.client.exception.MQClientException; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.future.FutureTaskExt; + import org.apache.rocketmq.common.thread.ThreadPoolMonitor; +@@ -51,10 +50,12 @@ import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager; + import org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline; + import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline; + import org.apache.rocketmq.remoting.ChannelEventListener; ++import org.apache.rocketmq.remoting.InvokeCallback; + import org.apache.rocketmq.remoting.RemotingServer; + import org.apache.rocketmq.remoting.netty.NettyRemotingServer; + import org.apache.rocketmq.remoting.netty.NettyServerConfig; + import org.apache.rocketmq.remoting.netty.RequestTask; ++import org.apache.rocketmq.remoting.netty.ResponseFuture; + import org.apache.rocketmq.remoting.netty.TlsSystemConfig; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; + import org.apache.rocketmq.remoting.protocol.RequestCode; +@@ -239,12 +240,21 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu + long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { +- this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, responseFuture -> { +- if (responseFuture.getResponseCommand() == null) { +- future.completeExceptionally(new MQClientException("response is null after send request to client", responseFuture.getCause())); +- return; ++ this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ future.complete(response); ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ future.completeExceptionally(throwable); + } +- future.complete(responseFuture.getResponseCommand()); + }); + } catch (Throwable t) { + future.completeExceptionally(t); +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +index 3f3a4ae40..e2d05b0f5 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +@@ -24,6 +24,7 @@ import java.util.ArrayList; + import java.util.List; + import java.util.Set; + import java.util.UUID; ++import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ThreadLocalRandom; + import java.util.concurrent.atomic.AtomicReference; + import java.util.stream.Collectors; +@@ -85,6 +86,7 @@ import static org.mockito.ArgumentMatchers.any; + import static org.mockito.ArgumentMatchers.anyLong; + import static org.mockito.ArgumentMatchers.anyString; + import static org.mockito.Mockito.doAnswer; ++import static org.mockito.Mockito.doReturn; + + @RunWith(MockitoJUnitRunner.class) + public class MQClientAPIExtTest { +@@ -109,13 +111,9 @@ public class MQClientAPIExtTest { + + @Test + public void testSendHeartbeatAsync() throws Exception { +- doAnswer((Answer) mock -> { +- InvokeCallback invokeCallback = mock.getArgument(3); +- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); +- responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); +- invokeCallback.operationComplete(responseFuture); +- return null; +- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ CompletableFuture future = new CompletableFuture<>(); ++ future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); ++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + assertNotNull(mqClientAPI.sendHeartbeatAsync(BROKER_ADDR, new HeartbeatData(), TIMEOUT).get()); + } +@@ -123,20 +121,16 @@ public class MQClientAPIExtTest { + @Test + public void testSendMessageAsync() throws Exception { + AtomicReference msgIdRef = new AtomicReference<>(); +- doAnswer((Answer) mock -> { +- InvokeCallback invokeCallback = mock.getArgument(3); +- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); +- RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); +- SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); +- sendMessageResponseHeader.setMsgId(msgIdRef.get()); +- sendMessageResponseHeader.setQueueId(0); +- sendMessageResponseHeader.setQueueOffset(1L); +- response.setCode(ResponseCode.SUCCESS); +- response.makeCustomHeaderToNet(); +- responseFuture.putResponse(response); +- invokeCallback.operationComplete(responseFuture); +- return null; +- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ CompletableFuture future = new CompletableFuture<>(); ++ RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); ++ SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); ++ sendMessageResponseHeader.setMsgId(msgIdRef.get()); ++ sendMessageResponseHeader.setQueueId(0); ++ sendMessageResponseHeader.setQueueOffset(1L); ++ response.setCode(ResponseCode.SUCCESS); ++ response.makeCustomHeaderToNet(); ++ future.complete(response); ++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + MessageExt messageExt = createMessage(); + msgIdRef.set(MessageClientIDSetter.getUniqID(messageExt)); +@@ -150,20 +144,16 @@ public class MQClientAPIExtTest { + + @Test + public void testSendMessageListAsync() throws Exception { +- doAnswer((Answer) mock -> { +- InvokeCallback invokeCallback = mock.getArgument(3); +- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); +- RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); +- SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); +- sendMessageResponseHeader.setMsgId(""); +- sendMessageResponseHeader.setQueueId(0); +- sendMessageResponseHeader.setQueueOffset(1L); +- response.setCode(ResponseCode.SUCCESS); +- response.makeCustomHeaderToNet(); +- responseFuture.putResponse(response); +- invokeCallback.operationComplete(responseFuture); +- return null; +- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ CompletableFuture future = new CompletableFuture<>(); ++ RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); ++ SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); ++ sendMessageResponseHeader.setMsgId(""); ++ sendMessageResponseHeader.setQueueId(0); ++ sendMessageResponseHeader.setQueueOffset(1L); ++ response.setCode(ResponseCode.SUCCESS); ++ response.makeCustomHeaderToNet(); ++ future.complete(response); ++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + List messageExtList = new ArrayList<>(); + StringBuilder sb = new StringBuilder(); +@@ -182,13 +172,9 @@ public class MQClientAPIExtTest { + + @Test + public void testSendMessageBackAsync() throws Exception { +- doAnswer((Answer) mock -> { +- InvokeCallback invokeCallback = mock.getArgument(3); +- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); +- responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); +- invokeCallback.operationComplete(responseFuture); +- return null; +- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ CompletableFuture future = new CompletableFuture<>(); ++ future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); ++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + RemotingCommand remotingCommand = mqClientAPI.sendMessageBackAsync(BROKER_ADDR, new ConsumerSendMsgBackRequestHeader(), TIMEOUT) + .get(); +@@ -285,7 +271,7 @@ public class MQClientAPIExtTest { + body.setConsumerIdList(clientIds); + response.setBody(body.encode()); + responseFuture.putResponse(response); +- invokeCallback.operationComplete(responseFuture); ++ invokeCallback.operationSucceed(responseFuture.getResponseCommand()); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + +@@ -302,7 +288,7 @@ public class MQClientAPIExtTest { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.makeCustomHeaderToNet(); + responseFuture.putResponse(response); +- invokeCallback.operationComplete(responseFuture); ++ invokeCallback.operationSucceed(responseFuture.getResponseCommand()); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + +@@ -322,7 +308,7 @@ public class MQClientAPIExtTest { + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + responseFuture.putResponse(response); +- invokeCallback.operationComplete(responseFuture); ++ invokeCallback.operationSucceed(responseFuture.getResponseCommand()); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + +@@ -335,18 +321,15 @@ public class MQClientAPIExtTest { + @Test + public void testSearchOffsetAsync() throws Exception { + long offset = ThreadLocalRandom.current().nextLong(); +- doAnswer((Answer) mock -> { +- InvokeCallback invokeCallback = mock.getArgument(3); +- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); +- RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); +- SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); +- responseHeader.setOffset(offset); +- response.setCode(ResponseCode.SUCCESS); +- response.makeCustomHeaderToNet(); +- responseFuture.putResponse(response); +- invokeCallback.operationComplete(responseFuture); +- return null; +- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ CompletableFuture future = new CompletableFuture<>(); ++ RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); ++ SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); ++ responseHeader.setOffset(offset); ++ response.setCode(ResponseCode.SUCCESS); ++ response.makeCustomHeaderToNet(); ++ future.complete(response); ++ ++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader(); + requestHeader.setTopic(TOPIC); +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java +index ce78fa923..6be491745 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java +@@ -17,7 +17,22 @@ + package org.apache.rocketmq.remoting; + + import org.apache.rocketmq.remoting.netty.ResponseFuture; ++import org.apache.rocketmq.remoting.protocol.RemotingCommand; + + public interface InvokeCallback { ++ /** ++ * This method is expected to be invoked after {@link #operationSucceed(RemotingCommand)} ++ * or {@link #operationFail(Throwable)} ++ * ++ * @param responseFuture the returned object contains response or exception ++ */ + void operationComplete(final ResponseFuture responseFuture); ++ ++ default void operationSucceed(final RemotingCommand response) { ++ ++ } ++ ++ default void operationFail(final Throwable throwable) { ++ ++ } + } +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java +index ff0b3df95..c8389eedb 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java +@@ -20,11 +20,11 @@ import java.util.List; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ExecutorService; + import org.apache.rocketmq.remoting.exception.RemotingConnectException; +-import org.apache.rocketmq.remoting.exception.RemotingException; + import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; + import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; + import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; + import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; ++import org.apache.rocketmq.remoting.netty.ResponseFuture; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; + + public interface RemotingClient extends RemotingService { +@@ -51,18 +51,21 @@ public interface RemotingClient extends RemotingService { + final long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { +- invokeAsync(addr, request, timeoutMillis, responseFuture -> { +- RemotingCommand response = responseFuture.getResponseCommand(); +- if (response != null) { ++ invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + future.complete(response); +- } else { +- if (!responseFuture.isSendRequestOK()) { +- future.completeExceptionally(new RemotingSendRequestException(addr, responseFuture.getCause())); +- } else if (responseFuture.isTimeout()) { +- future.completeExceptionally(new RemotingTimeoutException(addr, timeoutMillis, responseFuture.getCause())); +- } else { +- future.completeExceptionally(new RemotingException(request.toString(), responseFuture.getCause())); +- } ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ future.completeExceptionally(throwable); + } + }); + } catch (Throwable t) { +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +index fce2de267..12e66f913 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +@@ -23,20 +23,23 @@ import io.netty.handler.ssl.SslContext; + import io.netty.handler.ssl.SslHandler; + import io.netty.util.concurrent.Future; + import io.opentelemetry.api.common.AttributesBuilder; +-import java.net.SocketAddress; + import java.util.ArrayList; + import java.util.HashMap; + import java.util.Iterator; + import java.util.LinkedList; + import java.util.List; + import java.util.Map.Entry; ++import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; ++import java.util.concurrent.ExecutionException; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.RejectedExecutionException; + import java.util.concurrent.Semaphore; + import java.util.concurrent.TimeUnit; ++import java.util.concurrent.TimeoutException; ++import java.util.concurrent.atomic.AtomicReference; + import java.util.function.Consumer; + import javax.annotation.Nullable; + import org.apache.rocketmq.common.AbortProcessException; +@@ -125,7 +128,7 @@ public abstract class NettyRemotingAbstract { + * Constructor, specifying capacity of one-way and asynchronous semaphores. + * + * @param permitsOneway Number of permits for one-way requests. +- * @param permitsAsync Number of permits for asynchronous requests. ++ * @param permitsAsync Number of permits for asynchronous requests. + */ + public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) { + this.semaphoreOneway = new Semaphore(permitsOneway, true); +@@ -367,8 +370,7 @@ public abstract class NettyRemotingAbstract { + responseFuture.release(); + } + } else { +- log.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); +- log.warn(cmd.toString()); ++ log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + } + } + +@@ -467,57 +469,68 @@ public abstract class NettyRemotingAbstract { + public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request, + final long timeoutMillis) + throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException { +- //get the request id +- final int opaque = request.getOpaque(); +- + try { +- final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, null, null); +- this.responseTable.put(opaque, responseFuture); +- final SocketAddress addr = channel.remoteAddress(); +- channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { +- if (f.isSuccess()) { +- responseFuture.setSendRequestOK(true); +- return; +- } +- +- responseFuture.setSendRequestOK(false); +- responseTable.remove(opaque); +- responseFuture.setCause(f.cause()); +- responseFuture.putResponse(null); +- log.warn("Failed to write a request command to {}, caused by underlying I/O operation failure", addr); +- }); ++ return invokeImpl(channel, request, timeoutMillis).thenApply(ResponseFuture::getResponseCommand) ++ .get(timeoutMillis, TimeUnit.MILLISECONDS); ++ } catch (ExecutionException e) { ++ throw new RemotingSendRequestException(channel.remoteAddress().toString(), e.getCause()); ++ } catch (TimeoutException e) { ++ throw new RemotingTimeoutException(channel.remoteAddress().toString(), timeoutMillis, e.getCause()); ++ } ++ } + +- RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis); +- if (null == responseCommand) { +- if (responseFuture.isSendRequestOK()) { +- throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis, +- responseFuture.getCause()); +- } else { +- throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause()); +- } ++ public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand request, ++ final long timeoutMillis) { ++ String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); ++ doBeforeRpcHooks(channelRemoteAddr, request); ++ return invoke0(channel, request, timeoutMillis).whenComplete((v, t) -> { ++ if (t == null) { ++ doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand()); + } +- +- return responseCommand; +- } finally { +- this.responseTable.remove(opaque); +- } ++ }); + } + +- public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, +- final InvokeCallback invokeCallback) +- throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { ++ protected CompletableFuture invoke0(final Channel channel, final RemotingCommand request, ++ final long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); + long beginStartTime = System.currentTimeMillis(); + final int opaque = request.getOpaque(); +- boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); ++ ++ boolean acquired; ++ try { ++ acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ return future; ++ } + if (acquired) { + final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeoutMillis < costTime) { + once.release(); +- throw new RemotingTimeoutException("invokeAsyncImpl call timeout"); ++ future.completeExceptionally(new RemotingTimeoutException("invokeAsyncImpl call timeout")); ++ return future; + } + +- final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once); ++ AtomicReference responseFutureReference = new AtomicReference<>(); ++ final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, request, timeoutMillis - costTime, ++ new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ future.complete(responseFutureReference.get()); ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ future.completeExceptionally(throwable); ++ } ++ }, once); ++ responseFutureReference.set(responseFuture); + this.responseTable.put(opaque, responseFuture); + try { + channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { +@@ -528,15 +541,17 @@ public abstract class NettyRemotingAbstract { + requestFail(opaque); + log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); + }); ++ return future; + } catch (Exception e) { + responseTable.remove(opaque); + responseFuture.release(); + log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); +- throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); ++ future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e)); ++ return future; + } + } else { + if (timeoutMillis <= 0) { +- throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast"); ++ future.completeExceptionally(new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast")); + } else { + String info = + String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", +@@ -545,11 +560,31 @@ public abstract class NettyRemotingAbstract { + this.semaphoreAsync.availablePermits() + ); + log.warn(info); +- throw new RemotingTimeoutException(info); ++ future.completeExceptionally(new RemotingTimeoutException(info)); + } ++ return future; + } + } + ++ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, ++ final InvokeCallback invokeCallback) { ++ invokeImpl(channel, request, timeoutMillis) ++ .whenComplete((v, t) -> { ++ if (t == null) { ++ invokeCallback.operationComplete(v); ++ } else { ++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, timeoutMillis, null, null); ++ responseFuture.setCause(t); ++ invokeCallback.operationComplete(responseFuture); ++ } ++ }) ++ .thenAccept(responseFuture -> invokeCallback.operationSucceed(responseFuture.getResponseCommand())) ++ .exceptionally(t -> { ++ invokeCallback.operationFail(t); ++ return null; ++ }); ++ } ++ + private void requestFail(final int opaque) { + ResponseFuture responseFuture = responseTable.remove(opaque); + if (responseFuture != null) { +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +index 64621dd6c..d784351a5 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +@@ -527,15 +527,13 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + if (channel != null && channel.isActive()) { + long left = timeoutMillis; + try { +- doBeforeRpcHooks(channelRemoteAddr, request); + long costTime = System.currentTimeMillis() - beginStartTime; + left -= costTime; + if (left <= 0) { + throw new RemotingTimeoutException("invokeSync call the addr[" + channelRemoteAddr + "] timeout"); + } + RemotingCommand response = this.invokeSyncImpl(channel, request, left); +- doAfterRpcHooks(channelRemoteAddr, request, response); +- this.updateChannelLastResponseTime(addr); ++ updateChannelLastResponseTime(addr); + return response; + } catch (RemotingSendRequestException e) { + LOGGER.warn("invokeSync: send request exception, so close the channel[{}]", channelRemoteAddr); +@@ -727,18 +725,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + final Channel channel = this.getAndCreateChannel(addr); + String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); + if (channel != null && channel.isActive()) { +- try { +- doBeforeRpcHooks(channelRemoteAddr, request); +- long costTime = System.currentTimeMillis() - beginStartTime; +- if (timeoutMillis < costTime) { +- throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); +- } +- this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); +- } catch (RemotingSendRequestException e) { +- LOGGER.warn("invokeAsync: send request exception, so close the channel[{}]", channelRemoteAddr); +- this.closeChannel(addr, channel); +- throw e; ++ long costTime = System.currentTimeMillis() - beginStartTime; ++ if (timeoutMillis < costTime) { ++ throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); + } ++ this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); + } else { + this.closeChannel(addr, channel); + throw new RemotingConnectException(addr); +@@ -931,11 +922,19 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + + @Override + public void operationComplete(ResponseFuture responseFuture) { +- if (responseFuture != null && responseFuture.isSendRequestOK() && responseFuture.getResponseCommand() != null) { +- NettyRemotingClient.this.updateChannelLastResponseTime(addr); +- } + this.invokeCallback.operationComplete(responseFuture); + } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ updateChannelLastResponseTime(addr); ++ this.invokeCallback.operationSucceed(response); ++ } ++ ++ @Override ++ public void operationFail(final Throwable throwable) { ++ this.invokeCallback.operationFail(throwable); ++ } + } + + class NettyClientHandler extends SimpleChannelInboundHandler { +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java +index 19f705d74..0882818fe 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java +@@ -22,6 +22,9 @@ import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicBoolean; + import org.apache.rocketmq.remoting.InvokeCallback; + import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; ++import org.apache.rocketmq.remoting.exception.RemotingException; ++import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; ++import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; + + public class ResponseFuture { +@@ -59,6 +62,18 @@ public class ResponseFuture { + public void executeInvokeCallback() { + if (invokeCallback != null) { + if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) { ++ RemotingCommand response = getResponseCommand(); ++ if (response != null) { ++ invokeCallback.operationSucceed(response); ++ } else { ++ if (!isSendRequestOK()) { ++ invokeCallback.operationFail(new RemotingSendRequestException(channel.remoteAddress().toString(), getCause())); ++ } else if (isTimeout()) { ++ invokeCallback.operationFail(new RemotingTimeoutException(channel.remoteAddress().toString(), getTimeoutMillis(), getCause())); ++ } else { ++ invokeCallback.operationFail(new RemotingException(getRequestCommand().toString(), getCause())); ++ } ++ } + invokeCallback.operationComplete(this); + } + } +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java +index 133e0ed31..5328e8845 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java +@@ -160,31 +160,38 @@ public class RpcClientImpl implements RpcClient { + InvokeCallback callback = new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { +- RemotingCommand responseCommand = responseFuture.getResponseCommand(); +- if (responseCommand == null) { +- processFailedResponse(addr, requestCommand, responseFuture, rpcResponsePromise); +- return; +- } ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + try { +- switch (responseCommand.getCode()) { ++ switch (response.getCode()) { + case ResponseCode.SUCCESS: + case ResponseCode.PULL_NOT_FOUND: + case ResponseCode.PULL_RETRY_IMMEDIATELY: + case ResponseCode.PULL_OFFSET_MOVED: + PullMessageResponseHeader responseHeader = +- (PullMessageResponseHeader) responseCommand.decodeCommandCustomHeader(PullMessageResponseHeader.class); +- rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); ++ (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class); ++ rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody())); + default: +- RpcResponse rpcResponse = new RpcResponse(new RpcException(responseCommand.getCode(), "unexpected remote response code")); ++ RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), "unexpected remote response code")); + rpcResponsePromise.setSuccess(rpcResponse); + + } + } catch (Exception e) { +- String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand; +- RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); ++ String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + requestCommand; ++ RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); + rpcResponsePromise.setSuccess(rpcResponse); + } + } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ String errorMessage = "process failed. addr: " + addr + ". Request: " + requestCommand; ++ RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, throwable)); ++ rpcResponsePromise.setSuccess(rpcResponse); ++ } + }; + + this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback); +diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java +index 90072960b..d0da0eb2e 100644 +--- a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java ++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java +@@ -26,12 +26,12 @@ import org.apache.rocketmq.remoting.exception.RemotingConnectException; + import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; + import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; + import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; +-import org.apache.rocketmq.remoting.netty.ResponseFuture; +-import org.apache.rocketmq.remoting.netty.NettyServerConfig; + import org.apache.rocketmq.remoting.netty.NettyClientConfig; +-import org.apache.rocketmq.remoting.netty.NettyRemotingServer; + import org.apache.rocketmq.remoting.netty.NettyRemotingClient; ++import org.apache.rocketmq.remoting.netty.NettyRemotingServer; + import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; ++import org.apache.rocketmq.remoting.netty.NettyServerConfig; ++import org.apache.rocketmq.remoting.netty.ResponseFuture; + import org.apache.rocketmq.remoting.protocol.LanguageCode; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; + import org.junit.AfterClass; +@@ -40,7 +40,6 @@ import org.junit.Test; + + import static org.assertj.core.api.Assertions.assertThat; + import static org.junit.Assert.assertNotNull; +-import static org.junit.Assert.assertTrue; + + public class RemotingServerTest { + private static RemotingServer remotingServer; +@@ -122,10 +121,19 @@ public class RemotingServerTest { + remotingClient.invokeAsync("localhost:" + remotingServer.localListenPort(), request, 1000 * 3, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + latch.countDown(); +- assertTrue(responseFuture != null); +- assertThat(responseFuture.getResponseCommand().getLanguage()).isEqualTo(LanguageCode.JAVA); +- assertThat(responseFuture.getResponseCommand().getExtFields()).hasSize(2); ++ assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA); ++ assertThat(response.getExtFields()).hasSize(2); ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ + } + }); + latch.await(); +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java +similarity index 57% +rename from client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java +rename to remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java +index 80188832e..8ddcdf35d 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java ++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java +@@ -15,23 +15,14 @@ + * limitations under the License. + */ + +-package org.apache.rocketmq.client.impl; ++package org.apache.rocketmq.remoting.netty; + +-import org.apache.rocketmq.remoting.InvokeCallback; +-import org.apache.rocketmq.remoting.netty.ResponseFuture; +- +-public abstract class BaseInvokeCallback implements InvokeCallback { +- private final MQClientAPIImpl mqClientAPI; +- +- public BaseInvokeCallback(MQClientAPIImpl mqClientAPI) { +- this.mqClientAPI = mqClientAPI; +- } ++import io.netty.channel.ChannelFuture; ++import io.netty.channel.local.LocalChannel; + ++public class MockChannel extends LocalChannel { + @Override +- public void operationComplete(final ResponseFuture responseFuture) { +- mqClientAPI.execRpcHooksAfterRequest(responseFuture); +- onComplete(responseFuture); ++ public ChannelFuture writeAndFlush(Object msg) { ++ return new MockChannelPromise(MockChannel.this); + } +- +- public abstract void onComplete(final ResponseFuture responseFuture); + } +diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java +new file mode 100644 +index 000000000..9c3a35487 +--- /dev/null ++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java +@@ -0,0 +1,191 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.remoting.netty; ++ ++import io.netty.channel.Channel; ++import io.netty.channel.ChannelPromise; ++import io.netty.util.concurrent.Future; ++import io.netty.util.concurrent.GenericFutureListener; ++import java.util.concurrent.ExecutionException; ++import java.util.concurrent.TimeUnit; ++import java.util.concurrent.TimeoutException; ++import org.jetbrains.annotations.NotNull; ++ ++public class MockChannelPromise implements ChannelPromise { ++ protected Channel channel; ++ ++ public MockChannelPromise(Channel channel) { ++ this.channel = channel; ++ } ++ ++ @Override ++ public Channel channel() { ++ return channel; ++ } ++ ++ @Override ++ public ChannelPromise setSuccess(Void result) { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise setSuccess() { ++ return this; ++ } ++ ++ @Override ++ public boolean trySuccess() { ++ return false; ++ } ++ ++ @Override ++ public ChannelPromise setFailure(Throwable cause) { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise addListener(GenericFutureListener> listener) { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise addListeners(GenericFutureListener>... listeners) { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise removeListener(GenericFutureListener> listener) { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise removeListeners(GenericFutureListener>... listeners) { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise sync() throws InterruptedException { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise syncUninterruptibly() { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise await() throws InterruptedException { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise awaitUninterruptibly() { ++ return this; ++ } ++ ++ @Override ++ public ChannelPromise unvoid() { ++ return this; ++ } ++ ++ @Override ++ public boolean isVoid() { ++ return false; ++ } ++ ++ @Override ++ public boolean trySuccess(Void result) { ++ return false; ++ } ++ ++ @Override ++ public boolean tryFailure(Throwable cause) { ++ return false; ++ } ++ ++ @Override ++ public boolean setUncancellable() { ++ return false; ++ } ++ ++ @Override ++ public boolean isSuccess() { ++ return false; ++ } ++ ++ @Override ++ public boolean isCancellable() { ++ return false; ++ } ++ ++ @Override ++ public Throwable cause() { ++ return null; ++ } ++ ++ @Override ++ public boolean await(long timeout, TimeUnit unit) throws InterruptedException { ++ return false; ++ } ++ ++ @Override ++ public boolean await(long timeoutMillis) throws InterruptedException { ++ return false; ++ } ++ ++ @Override ++ public boolean awaitUninterruptibly(long timeout, TimeUnit unit) { ++ return false; ++ } ++ ++ @Override ++ public boolean awaitUninterruptibly(long timeoutMillis) { ++ return false; ++ } ++ ++ @Override ++ public Void getNow() { ++ return null; ++ } ++ ++ @Override ++ public boolean cancel(boolean mayInterruptIfRunning) { ++ return false; ++ } ++ ++ @Override ++ public boolean isCancelled() { ++ return false; ++ } ++ ++ @Override ++ public boolean isDone() { ++ return false; ++ } ++ ++ @Override ++ public Void get() throws InterruptedException, ExecutionException { ++ return null; ++ } ++ ++ @Override ++ public Void get(long timeout, ++ @NotNull java.util.concurrent.TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { ++ return null; ++ } ++} +diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java +index 8381c132b..dbbea86ea 100644 +--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java ++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java +@@ -39,9 +39,19 @@ public class NettyRemotingAbstractTest { + final Semaphore semaphore = new Semaphore(0); + ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { + @Override +- public void operationComplete(final ResponseFuture responseFuture) { ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + assertThat(semaphore.availablePermits()).isEqualTo(0); + } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ ++ } + }, new SemaphoreReleaseOnlyOnce(semaphore)); + + remotingAbstract.responseTable.putIfAbsent(1, responseFuture); +@@ -75,9 +85,19 @@ public class NettyRemotingAbstractTest { + final Semaphore semaphore = new Semaphore(0); + ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { + @Override +- public void operationComplete(final ResponseFuture responseFuture) { ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { + assertThat(semaphore.availablePermits()).isEqualTo(0); + } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ ++ } + }, new SemaphoreReleaseOnlyOnce(semaphore)); + + remotingAbstract.responseTable.putIfAbsent(1, responseFuture); +@@ -98,7 +118,18 @@ public class NettyRemotingAbstractTest { + // mock timeout + ResponseFuture responseFuture = new ResponseFuture(null, dummyId, -1000, new InvokeCallback() { + @Override +- public void operationComplete(final ResponseFuture responseFuture) { ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ + } + }, null); + remotingAbstract.responseTable.putIfAbsent(dummyId, responseFuture); +@@ -111,7 +142,22 @@ public class NettyRemotingAbstractTest { + final Semaphore semaphore = new Semaphore(0); + RemotingCommand request = RemotingCommand.createRequestCommand(1, null); + ResponseFuture responseFuture = new ResponseFuture(null, 1, request, 3000, +- responseFuture1 -> assertThat(semaphore.availablePermits()).isEqualTo(0), new SemaphoreReleaseOnlyOnce(semaphore)); ++ new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ ++ @Override ++ public void operationSucceed(RemotingCommand response) { ++ assertThat(semaphore.availablePermits()).isEqualTo(0); ++ } ++ ++ @Override ++ public void operationFail(Throwable throwable) { ++ ++ } ++ }, new SemaphoreReleaseOnlyOnce(semaphore)); + + remotingAbstract.responseTable.putIfAbsent(1, responseFuture); + RemotingCommand response = RemotingCommand.createResponseCommand(0, "Foo"); +diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +index 8fabbb21d..e72e7bd53 100644 +--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +@@ -16,10 +16,17 @@ + */ + package org.apache.rocketmq.remoting.netty; + ++import io.netty.channel.Channel; ++import io.netty.channel.ChannelFuture; ++import io.netty.channel.local.LocalChannel; + import java.util.concurrent.CompletableFuture; ++import java.util.concurrent.ExecutionException; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; ++import java.util.concurrent.Semaphore; + import org.apache.rocketmq.remoting.InvokeCallback; ++import org.apache.rocketmq.remoting.RPCHook; ++import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; + import org.apache.rocketmq.remoting.exception.RemotingConnectException; + import org.apache.rocketmq.remoting.exception.RemotingException; + import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +@@ -29,23 +36,33 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; + import org.apache.rocketmq.remoting.protocol.ResponseCode; + import org.junit.Test; + import org.junit.runner.RunWith; ++import org.mockito.Mock; + import org.mockito.Spy; + import org.mockito.junit.MockitoJUnitRunner; + + import static org.assertj.core.api.Assertions.assertThat; ++import static org.assertj.core.api.Assertions.assertThatThrownBy; + import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; + import static org.mockito.ArgumentMatchers.any; + import static org.mockito.ArgumentMatchers.anyLong; + import static org.mockito.ArgumentMatchers.anyString; ++import static org.mockito.ArgumentMatchers.eq; + import static org.mockito.Mockito.doAnswer; ++import static org.mockito.Mockito.doReturn; ++import static org.mockito.Mockito.mock; ++import static org.mockito.Mockito.never; ++import static org.mockito.Mockito.times; ++import static org.mockito.Mockito.verify; + + @RunWith(MockitoJUnitRunner.class) + public class NettyRemotingClientTest { + @Spy + private NettyRemotingClient remotingClient = new NettyRemotingClient(new NettyClientConfig()); ++ @Mock ++ private RPCHook rpcHookMock; + + @Test +- public void testSetCallbackExecutor() throws NoSuchFieldException, IllegalAccessException { ++ public void testSetCallbackExecutor() { + ExecutorService customized = Executors.newCachedThreadPool(); + remotingClient.setCallbackExecutor(customized); + assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized); +@@ -61,7 +78,7 @@ public class NettyRemotingClientTest { + InvokeCallback callback = invocation.getArgument(3); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(response); +- callback.operationComplete(responseFuture); ++ callback.operationSucceed(responseFuture.getResponseCommand()); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + +@@ -78,9 +95,7 @@ public class NettyRemotingClientTest { + response.setCode(ResponseCode.SUCCESS); + doAnswer(invocation -> { + InvokeCallback callback = invocation.getArgument(3); +- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); +- responseFuture.setSendRequestOK(false); +- callback.operationComplete(responseFuture); ++ callback.operationFail(new RemotingSendRequestException(null)); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + +@@ -97,8 +112,7 @@ public class NettyRemotingClientTest { + response.setCode(ResponseCode.SUCCESS); + doAnswer(invocation -> { + InvokeCallback callback = invocation.getArgument(3); +- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), -1L, null, null); +- callback.operationComplete(responseFuture); ++ callback.operationFail(new RemotingTimeoutException("")); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + +@@ -115,8 +129,7 @@ public class NettyRemotingClientTest { + response.setCode(ResponseCode.SUCCESS); + doAnswer(invocation -> { + InvokeCallback callback = invocation.getArgument(3); +- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); +- callback.operationComplete(responseFuture); ++ callback.operationFail(new RemotingException(null)); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + +@@ -134,4 +147,158 @@ public class NettyRemotingClientTest { + assertThat(e.getMessage()).contains(addr); + } + } ++ ++ @Test ++ public void testInvoke0() throws ExecutionException, InterruptedException { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++ Channel channel = new MockChannel() { ++ @Override ++ public ChannelFuture writeAndFlush(Object msg) { ++ ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); ++ responseFuture.setResponseCommand(response); ++ responseFuture.executeInvokeCallback(); ++ return super.writeAndFlush(msg); ++ } ++ }; ++ CompletableFuture future = remotingClient.invoke0(channel, request, 1000L); ++ assertThat(future.get().getResponseCommand()).isEqualTo(response); ++ } ++ ++ @Test ++ public void testInvoke0WithException() { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++ Channel channel = new MockChannel() { ++ @Override ++ public ChannelFuture writeAndFlush(Object msg) { ++ ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); ++ responseFuture.executeInvokeCallback(); ++ return super.writeAndFlush(msg); ++ } ++ }; ++ CompletableFuture future = remotingClient.invoke0(channel, request, 1000L); ++ assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingException.class); ++ } ++ ++ @Test ++ public void testInvokeSync() throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { ++ remotingClient.registerRPCHook(rpcHookMock); ++ ++ Channel channel = new LocalChannel(); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); ++ responseFuture.setResponseCommand(response); ++ CompletableFuture future = new CompletableFuture<>(); ++ future.complete(responseFuture); ++ ++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); ++ RemotingCommand actual = remotingClient.invokeSyncImpl(channel, request, 1000); ++ assertThat(actual).isEqualTo(response); ++ ++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); ++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); ++ } ++ ++ @Test ++ public void testInvokeAsync() { ++ remotingClient.registerRPCHook(rpcHookMock); ++ Channel channel = new LocalChannel(); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); ++ responseFuture.setResponseCommand(response); ++ CompletableFuture future = new CompletableFuture<>(); ++ future.complete(responseFuture); ++ ++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); ++ ++ InvokeCallback callback = mock(InvokeCallback.class); ++ remotingClient.invokeAsyncImpl(channel, request, 1000, callback); ++ verify(callback, times(1)).operationSucceed(eq(response)); ++ verify(callback, times(1)).operationComplete(eq(responseFuture)); ++ verify(callback, never()).operationFail(any()); ++ ++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); ++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); ++ } ++ ++ @Test ++ public void testInvokeAsyncFail() { ++ remotingClient.registerRPCHook(rpcHookMock); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); ++ ++ Channel channel = new LocalChannel(); ++ CompletableFuture future = new CompletableFuture<>(); ++ future.completeExceptionally(new RemotingException(null)); ++ ++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); ++ ++ InvokeCallback callback = mock(InvokeCallback.class); ++ remotingClient.invokeAsyncImpl(channel, request, 1000, callback); ++ verify(callback, never()).operationSucceed(any()); ++ verify(callback, times(1)).operationComplete(any()); ++ verify(callback, times(1)).operationFail(any()); ++ ++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); ++ verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); ++ } ++ ++ @Test ++ public void testInvokeImpl() throws ExecutionException, InterruptedException { ++ remotingClient.registerRPCHook(rpcHookMock); ++ Channel channel = new LocalChannel(); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++ ++ } ++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); ++ responseFuture.setResponseCommand(response); ++ CompletableFuture future = new CompletableFuture<>(); ++ future.complete(responseFuture); ++ ++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); ++ ++ CompletableFuture future0 = remotingClient.invokeImpl(channel, request, 1000); ++ assertThat(future0.get()).isEqualTo(responseFuture); ++ ++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); ++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); ++ } ++ ++ @Test ++ public void testInvokeImplFail() { ++ remotingClient.registerRPCHook(rpcHookMock); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); ++ ++ Channel channel = new LocalChannel(); ++ CompletableFuture future = new CompletableFuture<>(); ++ future.completeExceptionally(new RemotingException(null)); ++ ++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); ++ ++ assertThatThrownBy(() -> remotingClient.invokeImpl(channel, request, 1000).get()).getCause().isInstanceOf(RemotingException.class); ++ ++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); ++ verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); ++ } + } +-- +2.32.0.windows.2 + + +From b9ffe0f9576f68b8a37cf3e2f68051658ae5a9a2 Mon Sep 17 00:00:00 2001 +From: Zhouxiang Zhan +Date: Sun, 8 Oct 2023 16:33:44 +0800 +Subject: [PATCH 2/7] [ISSUE #7296] Add ChannelEventListener for + MQClientAPIImpl (#7324) + +* Add ChannelEventListener for MQClientAPIImpl + +* add heartbeat when channel connect + +* remove log + +* Add enableHeartbeatChannelEventListener for ClientConfig +--- + .../apache/rocketmq/client/ClientConfig.java | 55 ++++++++++++++----- + .../rocketmq/client/impl/MQClientAPIImpl.java | 9 ++- + .../client/impl/factory/MQClientInstance.java | 35 +++++++++++- + .../remoting/netty/NettyRemotingClient.java | 2 + + 4 files changed, 85 insertions(+), 16 deletions(-) + +diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +index bb0fe3522..f9843cc02 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +@@ -94,6 +94,8 @@ public class ClientConfig { + private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false")); + private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false")); + ++ private boolean enableHeartbeatChannelEventListener = true; ++ + public String buildMQClientId() { + StringBuilder sb = new StringBuilder(); + sb.append(this.getClientIP()); +@@ -201,6 +203,7 @@ public class ClientConfig { + this.useHeartbeatV2 = cc.useHeartbeatV2; + this.startDetectorEnable = cc.startDetectorEnable; + this.sendLatencyEnable = cc.sendLatencyEnable; ++ this.enableHeartbeatChannelEventListener = cc.enableHeartbeatChannelEventListener; + this.detectInterval = cc.detectInterval; + this.detectTimeout = cc.detectTimeout; + } +@@ -228,6 +231,7 @@ public class ClientConfig { + cc.enableStreamRequestType = enableStreamRequestType; + cc.useHeartbeatV2 = useHeartbeatV2; + cc.startDetectorEnable = startDetectorEnable; ++ cc.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; + cc.sendLatencyEnable = sendLatencyEnable; + cc.detectInterval = detectInterval; + cc.detectTimeout = detectTimeout; +@@ -418,6 +422,14 @@ public class ClientConfig { + this.startDetectorEnable = startDetectorEnable; + } + ++ public boolean isEnableHeartbeatChannelEventListener() { ++ return enableHeartbeatChannelEventListener; ++ } ++ ++ public void setEnableHeartbeatChannelEventListener(boolean enableHeartbeatChannelEventListener) { ++ this.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; ++ } ++ + public int getDetectTimeout() { + return this.detectTimeout; + } +@@ -444,19 +456,34 @@ public class ClientConfig { + + @Override + public String toString() { +- return "ClientConfig [namesrvAddr=" + namesrvAddr +- + ", clientIP=" + clientIP + ", instanceName=" + instanceName +- + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads +- + ", pollNameServerInterval=" + pollNameServerInterval +- + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval +- + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval +- + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException +- + ", unitMode=" + unitMode + ", unitName=" + unitName +- + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS +- + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name() +- + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout +- + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody +- + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable +- + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]"; ++ return "ClientConfig{" + ++ "namesrvAddr='" + namesrvAddr + '\'' + ++ ", clientIP='" + clientIP + '\'' + ++ ", instanceName='" + instanceName + '\'' + ++ ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + ++ ", namespace='" + namespace + '\'' + ++ ", namespaceInitialized=" + namespaceInitialized + ++ ", accessChannel=" + accessChannel + ++ ", pollNameServerInterval=" + pollNameServerInterval + ++ ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ++ ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval + ++ ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + ++ ", unitMode=" + unitMode + ++ ", unitName='" + unitName + '\'' + ++ ", decodeReadBody=" + decodeReadBody + ++ ", decodeDecompressBody=" + decodeDecompressBody + ++ ", vipChannelEnabled=" + vipChannelEnabled + ++ ", useHeartbeatV2=" + useHeartbeatV2 + ++ ", useTLS=" + useTLS + ++ ", socksProxyConfig='" + socksProxyConfig + '\'' + ++ ", mqClientApiTimeout=" + mqClientApiTimeout + ++ ", detectTimeout=" + detectTimeout + ++ ", detectInterval=" + detectInterval + ++ ", language=" + language + ++ ", enableStreamRequestType=" + enableStreamRequestType + ++ ", sendLatencyEnable=" + sendLatencyEnable + ++ ", startDetectorEnable=" + startDetectorEnable + ++ ", enableHeartbeatChannelEventListener=" + enableHeartbeatChannelEventListener + ++ '}'; + } + } +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +index 2407e5737..e152be811 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +@@ -79,6 +79,7 @@ import org.apache.rocketmq.common.sysflag.PullSysFlag; + import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.remoting.ChannelEventListener; + import org.apache.rocketmq.remoting.CommandCustomHeader; + import org.apache.rocketmq.remoting.InvokeCallback; + import org.apache.rocketmq.remoting.RPCHook; +@@ -246,10 +247,16 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { + public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, + final ClientRemotingProcessor clientRemotingProcessor, + RPCHook rpcHook, final ClientConfig clientConfig) { ++ this(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null); ++ } ++ ++ public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, ++ final ClientRemotingProcessor clientRemotingProcessor, ++ RPCHook rpcHook, final ClientConfig clientConfig, final ChannelEventListener channelEventListener) { + this.clientConfig = clientConfig; + topAddressing = new DefaultTopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName()); + topAddressing.registerChangeCallBack(this); +- this.remotingClient = new NettyRemotingClient(nettyClientConfig, null); ++ this.remotingClient = new NettyRemotingClient(nettyClientConfig, channelEventListener); + this.clientRemotingProcessor = clientRemotingProcessor; + + // Inject stream rpc hook first to make reserve field signature +diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +index 9484b26f8..09534a176 100644 +--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +@@ -16,6 +16,7 @@ + */ + package org.apache.rocketmq.client.impl.factory; + ++import io.netty.channel.Channel; + import java.util.Collections; + import java.util.HashMap; + import java.util.HashSet; +@@ -65,6 +66,7 @@ import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.message.MessageQueue; + import org.apache.rocketmq.common.message.MessageQueueAssignment; + import org.apache.rocketmq.common.topic.TopicValidator; ++import org.apache.rocketmq.remoting.ChannelEventListener; + import org.apache.rocketmq.remoting.RPCHook; + import org.apache.rocketmq.remoting.common.HeartbeatV2Result; + import org.apache.rocketmq.remoting.exception.RemotingException; +@@ -151,7 +153,38 @@ public class MQClientInstance { + this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS()); + this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig()); + ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this); +- this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig); ++ ChannelEventListener channelEventListener; ++ if (clientConfig.isEnableHeartbeatChannelEventListener()) { ++ channelEventListener = new ChannelEventListener() { ++ private final ConcurrentMap> brokerAddrTable = MQClientInstance.this.brokerAddrTable; ++ @Override ++ public void onChannelConnect(String remoteAddr, Channel channel) { ++ for (Map.Entry> addressEntry : brokerAddrTable.entrySet()) { ++ for (String address : addressEntry.getValue().values()) { ++ if (address.equals(remoteAddr)) { ++ sendHeartbeatToAllBrokerWithLockV2(false); ++ break; ++ } ++ } ++ } ++ } ++ ++ @Override ++ public void onChannelClose(String remoteAddr, Channel channel) { ++ } ++ ++ @Override ++ public void onChannelException(String remoteAddr, Channel channel) { ++ } ++ ++ @Override ++ public void onChannelIdle(String remoteAddr, Channel channel) { ++ } ++ }; ++ } else { ++ channelEventListener = null; ++ } ++ this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, channelEventListener); + + if (this.clientConfig.getNamesrvAddr() != null) { + this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr()); +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +index d784351a5..8631d0447 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +@@ -229,6 +229,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + handler.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); + } + ++ nettyEventExecutor.start(); ++ + TimerTask timerTaskScanResponseTable = new TimerTask() { + @Override + public void run(Timeout timeout) { +-- +2.32.0.windows.2 + + +From 3808387e1389278edbe4ef023d200ecb3015622b Mon Sep 17 00:00:00 2001 +From: lk +Date: Mon, 9 Oct 2023 16:07:56 +0800 +Subject: [PATCH 3/7] [ISSUE #7429] clean channel map when CLIENT_UNREGISTER in + proxy + +--- + .../service/sysmessage/HeartbeatSyncer.java | 31 ++++++--- + .../sysmessage/HeartbeatSyncerTest.java | 68 +++++++++++++++++++ + 2 files changed, 88 insertions(+), 11 deletions(-) + +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java +index f70c06b8f..fee3ea87d 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java +@@ -18,6 +18,7 @@ + package org.apache.rocketmq.proxy.service.sysmessage; + + import com.alibaba.fastjson.JSON; ++import io.netty.channel.Channel; + import java.nio.charset.StandardCharsets; + import java.util.List; + import java.util.Map; +@@ -73,16 +74,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { + ); + this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() { + @Override +- public void handle(ConsumerGroupEvent event, String s, Object... args) { +- if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { +- if (args == null || args.length < 1) { +- return; +- } +- if (args[0] instanceof ClientChannelInfo) { +- ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; +- remoteChannelMap.remove(clientChannelInfo.getChannel().id().asLongText()); +- } +- } ++ public void handle(ConsumerGroupEvent event, String group, Object... args) { ++ processConsumerGroupEvent(event, group, args); + } + + @Override +@@ -98,6 +91,18 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { + super.shutdown(); + } + ++ protected void processConsumerGroupEvent(ConsumerGroupEvent event, String group, Object... args) { ++ if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { ++ if (args == null || args.length < 1) { ++ return; ++ } ++ if (args[0] instanceof ClientChannelInfo) { ++ ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; ++ remoteChannelMap.remove(buildKey(group, clientChannelInfo.getChannel())); ++ } ++ } ++ } ++ + public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo, + ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, + Set subList) { +@@ -189,7 +194,7 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { + } + + RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData()); +- RemoteChannel channel = remoteChannelMap.computeIfAbsent(data.getGroup() + "@" + decodedChannel.id().asLongText(), key -> decodedChannel); ++ RemoteChannel channel = remoteChannelMap.computeIfAbsent(buildKey(data.getGroup(), decodedChannel), key -> decodedChannel); + channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute()); + ClientChannelInfo clientChannelInfo = new ClientChannelInfo( + channel, +@@ -228,4 +233,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { + // use local address, remoting port and grpc port to build unique local proxy Id + return proxyConfig.getLocalServeAddr() + "%" + proxyConfig.getRemotingListenPort() + "%" + proxyConfig.getGrpcServerPort(); + } ++ ++ private static String buildKey(String group, Channel channel) { ++ return group + "@" + channel.id().asLongText(); ++ } + } +diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +index 43fba3d03..9a2c5e343 100644 +--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +@@ -27,6 +27,7 @@ import com.google.common.collect.Sets; + import io.netty.channel.Channel; + import io.netty.channel.ChannelId; + import java.time.Duration; ++import java.util.Collections; + import java.util.HashMap; + import java.util.HashSet; + import java.util.List; +@@ -35,6 +36,7 @@ import java.util.concurrent.CompletableFuture; + import java.util.stream.Collectors; + import org.apache.commons.lang3.RandomStringUtils; + import org.apache.rocketmq.broker.client.ClientChannelInfo; ++import org.apache.rocketmq.broker.client.ConsumerGroupEvent; + import org.apache.rocketmq.broker.client.ConsumerManager; + import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt; + import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; +@@ -320,6 +322,72 @@ public class HeartbeatSyncerTest extends InitConfigTest { + } + } + ++ @Test ++ public void testProcessConsumerGroupEventForRemoting() { ++ String consumerGroup = "consumerGroup"; ++ Channel channel = createMockChannel(); ++ RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class); ++ RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, Collections.emptySet()); ++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo( ++ remotingChannel, ++ clientId, ++ LanguageCode.JAVA, ++ 4 ++ ); ++ ++ testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); ++ } ++ ++ @Test ++ public void testProcessConsumerGroupEventForGrpcV2() { ++ String consumerGroup = "consumerGroup"; ++ GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class); ++ GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class); ++ GrpcClientChannel grpcClientChannel = new GrpcClientChannel( ++ proxyRelayService, grpcClientSettingsManager, grpcChannelManager, ++ ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress), ++ clientId); ++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo( ++ grpcClientChannel, ++ clientId, ++ LanguageCode.JAVA, ++ 5 ++ ); ++ ++ testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); ++ } ++ ++ private void testProcessConsumerGroupEvent(String consumerGroup, ClientChannelInfo clientChannelInfo) { ++ HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null); ++ SendResult okSendResult = new SendResult(); ++ okSendResult.setSendStatus(SendStatus.SEND_OK); ++ ++ ArgumentCaptor messageArgumentCaptor = ArgumentCaptor.forClass(Message.class); ++ doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt) ++ .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong()); ++ ++ heartbeatSyncer.onConsumerRegister( ++ consumerGroup, ++ clientChannelInfo, ++ ConsumeType.CONSUME_PASSIVELY, ++ MessageModel.CLUSTERING, ++ ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, ++ Collections.emptySet() ++ ); ++ await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 1); ++ ++ // change local serve addr, to simulate other proxy receive messages ++ heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10); ++ ArgumentCaptor channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class); ++ doReturn(true).when(consumerManager).registerConsumer(anyString(), channelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean()); ++ ++ heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null); ++ assertEquals(1, heartbeatSyncer.remoteChannelMap.size()); ++ ++ heartbeatSyncer.processConsumerGroupEvent(ConsumerGroupEvent.CLIENT_UNREGISTER, consumerGroup, channelInfoArgumentCaptor.getValue()); ++ assertTrue(heartbeatSyncer.remoteChannelMap.isEmpty()); ++ } ++ + private MessageExt convertFromMessage(Message message) { + MessageExt messageExt = new MessageExt(); + messageExt.setTopic(message.getTopic()); +-- +2.32.0.windows.2 + + +From 0027a1486d4f2d6f7dce3010751167e883783945 Mon Sep 17 00:00:00 2001 +From: redlsz +Date: Mon, 9 Oct 2023 16:52:10 +0800 +Subject: [PATCH 4/7] [ISSUE #7412] Fix pop revive message error when reput + checkpoint + +--- + .../org/apache/rocketmq/broker/processor/PopReviveService.java | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +index 93167db37..d5174d3d1 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +@@ -595,6 +595,7 @@ public class PopReviveService extends ServiceThread { + newCk.setCId(oldCK.getCId()); + newCk.setTopic(oldCK.getTopic()); + newCk.setQueueId(oldCK.getQueueId()); ++ newCk.setBrokerName(oldCK.getBrokerName()); + newCk.addDiff(0); + MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId); + brokerController.getMessageStore().putMessage(ckMsg); +-- +2.32.0.windows.2 + + +From b18e564addbcff50165a5e1d9d4ab7db789d901b Mon Sep 17 00:00:00 2001 +From: rongtong +Date: Mon, 9 Oct 2023 21:43:01 +0800 +Subject: [PATCH 5/7] [ISSUE #7431] Fix flaky test of + DLedgerControllerTest#testBrokerLifecycleListener (#7432) + +* Fix flaky test of DLedgerControllerTest#testBrokerLifecycleListener +--- + .../impl/DLedgerControllerTest.java | 26 ++++++++++++------- + 1 file changed, 17 insertions(+), 9 deletions(-) + +diff --git a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java +index 595a5cb65..d6e5449c5 100644 +--- a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java ++++ b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java +@@ -63,7 +63,8 @@ public class DLedgerControllerTest { + private List baseDirs; + private List controllers; + +- public DLedgerController launchController(final String group, final String peers, final String selfId, final boolean isEnableElectUncleanMaster) { ++ public DLedgerController launchController(final String group, final String peers, final String selfId, ++ final boolean isEnableElectUncleanMaster) { + String tmpdir = System.getProperty("java.io.tmpdir"); + final String path = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + group + File.separator + selfId; + baseDirs.add(path); +@@ -121,11 +122,11 @@ public class DLedgerControllerTest { + final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, nextBrokerId, brokerAddress); + RemotingCommand remotingCommand2 = leader.registerBroker(registerBrokerToControllerRequestHeader).get(2, TimeUnit.SECONDS); + +- + assertEquals(ResponseCode.SUCCESS, remotingCommand2.getCode()); + } + +- public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, Long brokerId, ++ public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, ++ Long brokerId, + boolean exceptSuccess) throws Exception { + final ElectMasterRequestHeader electMasterRequestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); + RemotingCommand command = leader.electMaster(electMasterRequestHeader).get(2, TimeUnit.SECONDS); +@@ -186,9 +187,9 @@ public class DLedgerControllerTest { + registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L); + registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L); + // try elect +- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L,true); +- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); +- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L,false); ++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, true); ++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); ++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, false); + final RemotingCommand getInfoResponse = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS); + final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) getInfoResponse.readCustomHeader(); + assertEquals(1, replicaInfo.getMasterEpoch().intValue()); +@@ -239,6 +240,8 @@ public class DLedgerControllerTest { + @Test + public void testBrokerLifecycleListener() throws Exception { + final DLedgerController leader = mockMetaData(false); ++ ++ assertTrue(leader.isLeaderState()); + // Mock that master broker has been inactive, and try to elect a new master from sync-state-set + // But we shut down two controller, so the ElectMasterEvent will be appended to DLedger failed. + // So the statemachine still keep the stale master's information +@@ -247,15 +250,20 @@ public class DLedgerControllerTest { + dLedgerController.shutdown(); + controllers.remove(dLedgerController); + } ++ + final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME); + setBrokerElectPolicy(leader, 1L); + Exception exception = null; ++ RemotingCommand remotingCommand = null; + try { +- leader.electMaster(request).get(5, TimeUnit.SECONDS); ++ remotingCommand = leader.electMaster(request).get(5, TimeUnit.SECONDS); + } catch (Exception e) { + exception = e; + } +- assertNotNull(exception); ++ ++ assertTrue(exception != null || ++ remotingCommand != null && remotingCommand.getCode() == ResponseCode.CONTROLLER_NOT_LEADER); ++ + // Shut down leader controller + leader.shutdown(); + controllers.remove(leader); +@@ -272,7 +280,7 @@ public class DLedgerControllerTest { + setBrokerAlivePredicate(newLeader, 1L); + // Check if the statemachine is stale + final RemotingCommand resp = newLeader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)). +- get(10, TimeUnit.SECONDS); ++ get(10, TimeUnit.SECONDS); + final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader(); + assertEquals(1, replicaInfo.getMasterBrokerId().longValue()); + assertEquals(1, replicaInfo.getMasterEpoch().intValue()); +-- +2.32.0.windows.2 + + +From 38d3d5d95d371ac89f7d491a4c8719b4a22c60e1 Mon Sep 17 00:00:00 2001 +From: mxsm +Date: Tue, 10 Oct 2023 09:37:04 +0800 +Subject: [PATCH 6/7] [ISSUE #7433]Update the version in the README.md document + to 5.1.4 (#7434) + +--- + README.md | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/README.md b/README.md +index 56d253ce1..5aaa2ba73 100644 +--- a/README.md ++++ b/README.md +@@ -49,21 +49,21 @@ $ java -version + java version "1.8.0_121" + ``` + +-For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip) to download the 5.1.3 RocketMQ binary release, ++For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip) to download the 5.1.4 RocketMQ binary release, + unpack it to your local disk, such as `D:\rocketmq`. + For macOS and Linux users, execute following commands: + + ```shell + # Download release from the Apache mirror +-$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip ++$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip + + # Unpack the release +-$ unzip rocketmq-all-5.1.3-bin-release.zip ++$ unzip rocketmq-all-5.1.4-bin-release.zip + ``` + + Prepare a terminal and change to the extracted `bin` directory: + ```shell +-$ cd rocketmq-all-5.1.3-bin-release/bin ++$ cd rocketmq-all-5.1.4-bin-release/bin + ``` + + **1) Start NameServer** +-- +2.32.0.windows.2 + + +From 4acb43ecee03e429d036e3ff4c28bd402d1b30c7 Mon Sep 17 00:00:00 2001 +From: Zhouxiang Zhan +Date: Tue, 10 Oct 2023 13:54:01 +0800 +Subject: [PATCH 7/7] [ISSUE #7330] Add goaway and reconnection mechanism + (#7331) + +* Add shutdown wait for NettyRemotingServer + +* Add goaway and reconnection mechanism + +* Add client version check + +* Add enableTransparentRetry for NettyClientConfig + +* Add enableReconnectForGoAway for NettyClientConfig + +* fix unit test + +* fix client version check +--- + .../remoting/netty/NettyClientConfig.java | 30 ++++ + .../remoting/netty/NettyRemotingAbstract.java | 15 ++ + .../remoting/netty/NettyRemotingClient.java | 153 ++++++++++++++++-- + .../remoting/netty/NettyRemotingServer.java | 31 ++-- + .../remoting/netty/NettyServerConfig.java | 19 +++ + .../remoting/protocol/ResponseCode.java | 2 + + .../netty/NettyRemotingClientTest.java | 39 ++--- + 7 files changed, 239 insertions(+), 50 deletions(-) + +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java +index b2e7df754..c28288786 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java +@@ -53,6 +53,12 @@ public class NettyClientConfig { + private boolean disableCallbackExecutor = false; + private boolean disableNettyWorkerGroup = false; + ++ private long maxReconnectIntervalTimeSeconds = 60; ++ ++ private boolean enableReconnectForGoAway = true; ++ ++ private boolean enableTransparentRetry = true; ++ + public boolean isClientCloseSocketIfTimeout() { + return clientCloseSocketIfTimeout; + } +@@ -181,6 +187,30 @@ public class NettyClientConfig { + this.disableNettyWorkerGroup = disableNettyWorkerGroup; + } + ++ public long getMaxReconnectIntervalTimeSeconds() { ++ return maxReconnectIntervalTimeSeconds; ++ } ++ ++ public void setMaxReconnectIntervalTimeSeconds(long maxReconnectIntervalTimeSeconds) { ++ this.maxReconnectIntervalTimeSeconds = maxReconnectIntervalTimeSeconds; ++ } ++ ++ public boolean isEnableReconnectForGoAway() { ++ return enableReconnectForGoAway; ++ } ++ ++ public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) { ++ this.enableReconnectForGoAway = enableReconnectForGoAway; ++ } ++ ++ public boolean isEnableTransparentRetry() { ++ return enableTransparentRetry; ++ } ++ ++ public void setEnableTransparentRetry(boolean enableTransparentRetry) { ++ this.enableTransparentRetry = enableTransparentRetry; ++ } ++ + public String getSocksProxyConfig() { + return socksProxyConfig; + } +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +index 12e66f913..07ace28ea 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +@@ -40,9 +40,11 @@ import java.util.concurrent.Semaphore; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.TimeoutException; + import java.util.concurrent.atomic.AtomicReference; ++import java.util.concurrent.atomic.AtomicBoolean; + import java.util.function.Consumer; + import javax.annotation.Nullable; + import org.apache.rocketmq.common.AbortProcessException; ++import org.apache.rocketmq.common.MQVersion; + import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.ServiceThread; + import org.apache.rocketmq.common.UtilAll; +@@ -60,6 +62,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; + import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; + import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; ++import org.apache.rocketmq.remoting.protocol.ResponseCode; + + import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_IS_LONG_POLLING; + import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE; +@@ -120,6 +123,8 @@ public abstract class NettyRemotingAbstract { + */ + protected List rpcHooks = new ArrayList<>(); + ++ protected AtomicBoolean isShuttingDown = new AtomicBoolean(false); ++ + static { + NettyLogger.initNettyLogger(); + } +@@ -264,6 +269,16 @@ public abstract class NettyRemotingAbstract { + + Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque); + ++ if (isShuttingDown.get()) { ++ if (cmd.getVersion() > MQVersion.Version.V5_1_4.ordinal()) { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY, ++ "please go away"); ++ response.setOpaque(opaque); ++ writeResponse(ctx.channel(), cmd, response); ++ return; ++ } ++ } ++ + if (pair.getObject1().rejectRequest()) { + final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY, + "[REJECTREQUEST]system busy, start flow control for a while"); +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +index 8631d0447..4bc51bd83 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +@@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty; + + import com.alibaba.fastjson.JSON; + import com.alibaba.fastjson.TypeReference; ++import com.google.common.base.Stopwatch; + import io.netty.bootstrap.Bootstrap; + import io.netty.buffer.PooledByteBufAllocator; + import io.netty.channel.Channel; +@@ -48,6 +49,7 @@ import java.io.IOException; + import java.net.InetSocketAddress; + import java.net.SocketAddress; + import java.security.cert.CertificateException; ++import java.time.Duration; + import java.util.ArrayList; + import java.util.Collections; + import java.util.HashMap; +@@ -57,6 +59,7 @@ import java.util.Map; + import java.util.Random; + import java.util.Set; + import java.util.concurrent.ArrayBlockingQueue; ++import java.util.concurrent.CompletableFuture; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.ExecutorService; +@@ -66,6 +69,7 @@ import java.util.concurrent.atomic.AtomicInteger; + import java.util.concurrent.atomic.AtomicReference; + import java.util.concurrent.locks.Lock; + import java.util.concurrent.locks.ReentrantLock; ++import java.util.concurrent.locks.ReentrantReadWriteLock; + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.ThreadFactoryImpl; +@@ -82,6 +86,7 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; + import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; + import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++import org.apache.rocketmq.remoting.protocol.ResponseCode; + import org.apache.rocketmq.remoting.proxy.SocksProxyConfig; + + public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { +@@ -97,6 +102,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + private final Map proxyMap = new HashMap<>(); + private final ConcurrentHashMap bootstrapMap = new ConcurrentHashMap<>(); + private final ConcurrentMap channelTables = new ConcurrentHashMap<>(); ++ private final ConcurrentMap channelWrapperTables = new ConcurrentHashMap<>(); + + private final HashedWheelTimer timer = new HashedWheelTimer(r -> new Thread(r, "ClientHouseKeepingService")); + +@@ -356,9 +362,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + this.timer.stop(); + + for (String addr : this.channelTables.keySet()) { +- this.closeChannel(addr, this.channelTables.get(addr).getChannel()); ++ this.channelTables.get(addr).close(); + } + ++ this.channelWrapperTables.clear(); + this.channelTables.clear(); + + this.eventLoopGroupWorker.shutdownGracefully(); +@@ -416,7 +423,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + } + + if (removeItemFromTable) { +- this.channelTables.remove(addrRemote); ++ ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); ++ if (channelWrapper != null && channelWrapper.tryClose(channel)) { ++ this.channelTables.remove(addrRemote); ++ } + LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + } + +@@ -463,7 +473,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + } + + if (removeItemFromTable) { +- this.channelTables.remove(addrRemote); ++ ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); ++ if (channelWrapper != null && channelWrapper.tryClose(channel)) { ++ this.channelTables.remove(addrRemote); ++ } + LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + RemotingHelper.closeChannel(channel); + } +@@ -511,7 +524,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + if (addr.contains(namesrvAddr)) { + ChannelWrapper channelWrapper = this.channelTables.get(addr); + if (channelWrapper != null) { +- closeChannel(channelWrapper.getChannel()); ++ channelWrapper.close(); + } + } + } +@@ -689,8 +702,9 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + ChannelFuture channelFuture = fetchBootstrap(addr) + .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); + LOGGER.info("createChannel: begin to connect remote host[{}] asynchronously", addr); +- cw = new ChannelWrapper(channelFuture); ++ cw = new ChannelWrapper(addr, channelFuture); + this.channelTables.put(addr, cw); ++ this.channelWrapperTables.put(channelFuture.channel(), cw); + } + } catch (Exception e) { + LOGGER.error("createChannel: create channel exception", e); +@@ -758,6 +772,64 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + } + } + ++ @Override ++ public CompletableFuture invoke(String addr, RemotingCommand request, ++ long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++ final Channel channel = this.getAndCreateChannel(addr); ++ if (channel != null && channel.isActive()) { ++ return invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> { ++ if (t == null) { ++ updateChannelLastResponseTime(addr); ++ } ++ }).thenApply(ResponseFuture::getResponseCommand); ++ } else { ++ this.closeChannel(addr, channel); ++ future.completeExceptionally(new RemotingConnectException(addr)); ++ } ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++ return future; ++ } ++ ++ @Override ++ public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand request, ++ final long timeoutMillis) { ++ Stopwatch stopwatch = Stopwatch.createStarted(); ++ return super.invokeImpl(channel, request, timeoutMillis).thenCompose(responseFuture -> { ++ RemotingCommand response = responseFuture.getResponseCommand(); ++ if (response.getCode() == ResponseCode.GO_AWAY) { ++ if (nettyClientConfig.isEnableReconnectForGoAway()) { ++ ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> { ++ try { ++ if (channelWrapper0.reconnect()) { ++ LOGGER.info("Receive go away from channel {}, recreate the channel", channel0); ++ channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0); ++ } ++ } catch (Throwable t) { ++ LOGGER.error("Channel {} reconnect error", channelWrapper0, t); ++ } ++ return channelWrapper0; ++ }); ++ if (channelWrapper != null) { ++ if (nettyClientConfig.isEnableTransparentRetry()) { ++ long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); ++ stopwatch.stop(); ++ RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); ++ Channel retryChannel = channelWrapper.getChannel(); ++ if (channel != retryChannel) { ++ return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration); ++ } ++ } ++ } ++ } ++ } ++ return CompletableFuture.completedFuture(responseFuture); ++ }); ++ } ++ + @Override + public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) { + ExecutorService executorThis = executor; +@@ -877,30 +949,41 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + } + } + +- static class ChannelWrapper { +- private final ChannelFuture channelFuture; ++ class ChannelWrapper { ++ private final ReentrantReadWriteLock lock; ++ private ChannelFuture channelFuture; + // only affected by sync or async request, oneway is not included. ++ private ChannelFuture channelToClose; + private long lastResponseTime; ++ private volatile long lastReconnectTimestamp = 0L; ++ private final String channelAddress; + +- public ChannelWrapper(ChannelFuture channelFuture) { ++ public ChannelWrapper(String address, ChannelFuture channelFuture) { ++ this.lock = new ReentrantReadWriteLock(); + this.channelFuture = channelFuture; + this.lastResponseTime = System.currentTimeMillis(); ++ this.channelAddress = address; + } + + public boolean isOK() { +- return this.channelFuture.channel() != null && this.channelFuture.channel().isActive(); ++ return getChannel() != null && getChannel().isActive(); + } + + public boolean isWritable() { +- return this.channelFuture.channel().isWritable(); ++ return getChannel().isWritable(); + } + + private Channel getChannel() { +- return this.channelFuture.channel(); ++ return getChannelFuture().channel(); + } + + public ChannelFuture getChannelFuture() { +- return channelFuture; ++ lock.readLock().lock(); ++ try { ++ return this.channelFuture; ++ } finally { ++ lock.readLock().unlock(); ++ } + } + + public long getLastResponseTime() { +@@ -910,6 +993,52 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + public void updateLastResponseTime() { + this.lastResponseTime = System.currentTimeMillis(); + } ++ ++ public boolean reconnect() { ++ if (lock.writeLock().tryLock()) { ++ try { ++ if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) { ++ channelToClose = channelFuture; ++ String[] hostAndPort = getHostAndPort(channelAddress); ++ channelFuture = fetchBootstrap(channelAddress) ++ .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); ++ lastReconnectTimestamp = System.currentTimeMillis(); ++ return true; ++ } ++ } finally { ++ lock.writeLock().unlock(); ++ } ++ } ++ return false; ++ } ++ ++ public boolean tryClose(Channel channel) { ++ try { ++ lock.readLock().lock(); ++ if (channelFuture != null) { ++ if (channelFuture.channel().equals(channel)) { ++ return true; ++ } ++ } ++ } finally { ++ lock.readLock().unlock(); ++ } ++ return false; ++ } ++ ++ public void close() { ++ try { ++ lock.writeLock().lock(); ++ if (channelFuture != null) { ++ closeChannel(channelFuture.channel()); ++ } ++ if (channelToClose != null) { ++ closeChannel(channelToClose.channel()); ++ } ++ } finally { ++ lock.writeLock().unlock(); ++ } ++ } + } + + class InvokeCallbackWrapper implements InvokeCallback { +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +index aa0d46542..735d36168 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +@@ -53,6 +53,19 @@ import io.netty.util.HashedWheelTimer; + import io.netty.util.Timeout; + import io.netty.util.TimerTask; + import io.netty.util.concurrent.DefaultEventExecutorGroup; ++import java.io.IOException; ++import java.net.InetSocketAddress; ++import java.security.cert.CertificateException; ++import java.time.Duration; ++import java.util.List; ++import java.util.NoSuchElementException; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++import java.util.concurrent.ExecutorService; ++import java.util.concurrent.Executors; ++import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.ThreadPoolExecutor; ++import java.util.concurrent.TimeUnit; + import org.apache.commons.collections.CollectionUtils; + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.common.Pair; +@@ -74,19 +87,6 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; + import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +-import java.io.IOException; +-import java.net.InetSocketAddress; +-import java.security.cert.CertificateException; +-import java.util.List; +-import java.util.NoSuchElementException; +-import java.util.concurrent.ConcurrentHashMap; +-import java.util.concurrent.ConcurrentMap; +-import java.util.concurrent.ExecutorService; +-import java.util.concurrent.Executors; +-import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ThreadPoolExecutor; +-import java.util.concurrent.TimeUnit; +- + @SuppressWarnings("NullableProblems") + public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer { + private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME); +@@ -305,6 +305,10 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti + @Override + public void shutdown() { + try { ++ if (nettyServerConfig.isEnableShutdownGracefully() && isShuttingDown.compareAndSet(false, true)) { ++ Thread.sleep(Duration.ofSeconds(nettyServerConfig.getShutdownWaitTimeSeconds()).toMillis()); ++ } ++ + this.timer.stop(); + + this.eventLoopGroupBoss.shutdownGracefully(); +@@ -736,6 +740,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti + + @Override + public void shutdown() { ++ isShuttingDown.set(true); + if (this.serverChannel != null) { + try { + this.serverChannel.close().await(5, TimeUnit.SECONDS); +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java +index 59ef2c84f..756661f62 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java +@@ -38,6 +38,9 @@ public class NettyServerConfig implements Cloneable { + private int serverSocketBacklog = NettySystemConfig.socketBacklog; + private boolean serverPooledByteBufAllocatorEnable = true; + ++ private boolean enableShutdownGracefully = false; ++ private int shutdownWaitTimeSeconds = 30; ++ + /** + * make install + * +@@ -171,4 +174,20 @@ public class NettyServerConfig implements Cloneable { + public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) { + this.writeBufferHighWaterMark = writeBufferHighWaterMark; + } ++ ++ public boolean isEnableShutdownGracefully() { ++ return enableShutdownGracefully; ++ } ++ ++ public void setEnableShutdownGracefully(boolean enableShutdownGracefully) { ++ this.enableShutdownGracefully = enableShutdownGracefully; ++ } ++ ++ public int getShutdownWaitTimeSeconds() { ++ return shutdownWaitTimeSeconds; ++ } ++ ++ public void setShutdownWaitTimeSeconds(int shutdownWaitTimeSeconds) { ++ this.shutdownWaitTimeSeconds = shutdownWaitTimeSeconds; ++ } + } +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java +index e81dadf2e..be945c48f 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java +@@ -99,6 +99,8 @@ public class ResponseCode extends RemotingSysResponseCode { + public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004; + public static final int RPC_TIME_OUT = -1006; + ++ public static final int GO_AWAY = 1500; ++ + /** + * Controller response code + */ +diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +index e72e7bd53..1cc6b4f46 100644 +--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +@@ -47,7 +47,6 @@ import static org.mockito.ArgumentMatchers.any; + import static org.mockito.ArgumentMatchers.anyLong; + import static org.mockito.ArgumentMatchers.anyString; + import static org.mockito.ArgumentMatchers.eq; +-import static org.mockito.Mockito.doAnswer; + import static org.mockito.Mockito.doReturn; + import static org.mockito.Mockito.mock; + import static org.mockito.Mockito.never; +@@ -74,13 +73,11 @@ public class NettyRemotingClientTest { + + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); +- doAnswer(invocation -> { +- InvokeCallback callback = invocation.getArgument(3); +- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); +- responseFuture.setResponseCommand(response); +- callback.operationSucceed(responseFuture.getResponseCommand()); +- return null; +- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++ responseFuture.setResponseCommand(response); ++ CompletableFuture future0 = new CompletableFuture<>(); ++ future0.complete(responseFuture.getResponseCommand()); ++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); + RemotingCommand actual = future.get(); +@@ -93,11 +90,9 @@ public class NettyRemotingClientTest { + + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); +- doAnswer(invocation -> { +- InvokeCallback callback = invocation.getArgument(3); +- callback.operationFail(new RemotingSendRequestException(null)); +- return null; +- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ CompletableFuture future0 = new CompletableFuture<>(); ++ future0.completeExceptionally(new RemotingSendRequestException(null)); ++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); + Throwable thrown = catchThrowable(future::get); +@@ -110,11 +105,9 @@ public class NettyRemotingClientTest { + + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); +- doAnswer(invocation -> { +- InvokeCallback callback = invocation.getArgument(3); +- callback.operationFail(new RemotingTimeoutException("")); +- return null; +- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ CompletableFuture future0 = new CompletableFuture<>(); ++ future0.completeExceptionally(new RemotingTimeoutException("")); ++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); + Throwable thrown = catchThrowable(future::get); +@@ -125,13 +118,9 @@ public class NettyRemotingClientTest { + public void testRemotingException() throws Exception { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + +- RemotingCommand response = RemotingCommand.createResponseCommand(null); +- response.setCode(ResponseCode.SUCCESS); +- doAnswer(invocation -> { +- InvokeCallback callback = invocation.getArgument(3); +- callback.operationFail(new RemotingException(null)); +- return null; +- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ CompletableFuture future0 = new CompletableFuture<>(); ++ future0.completeExceptionally(new RemotingException("")); ++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); + Throwable thrown = catchThrowable(future::get); +-- +2.32.0.windows.2 + diff --git a/patch021-backport-some-enhancements.patch b/patch021-backport-some-enhancements.patch new file mode 100644 index 000000000..839b65fea --- /dev/null +++ b/patch021-backport-some-enhancements.patch @@ -0,0 +1,344 @@ +From dc3f22ffe9eb83ace991b68921076093c7c0da5f Mon Sep 17 00:00:00 2001 +From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> +Date: Tue, 10 Oct 2023 17:39:23 +0800 +Subject: [PATCH 1/6] add getter for class Message ,fix json serialize bug + (#7439) + +Co-authored-by: LetLetMe +--- + .../rocketmq/common/message/Message.java | 24 ++++++++++++++++++- + 1 file changed, 23 insertions(+), 1 deletion(-) + +diff --git a/common/src/main/java/org/apache/rocketmq/common/message/Message.java b/common/src/main/java/org/apache/rocketmq/common/message/Message.java +index e02b526a1..c7997c473 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/message/Message.java ++++ b/common/src/main/java/org/apache/rocketmq/common/message/Message.java +@@ -218,14 +218,36 @@ public class Message implements Serializable { + public void setDelayTimeSec(long sec) { + this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC, String.valueOf(sec)); + } ++ ++ public long getDelayTimeSec() { ++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC); ++ if (t != null) { ++ return Long.parseLong(t); ++ } ++ return 0; ++ } ++ + public void setDelayTimeMs(long timeMs) { + this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_MS, String.valueOf(timeMs)); + } ++ ++ public long getDelayTimeMs() { ++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS); ++ if (t != null) { ++ return Long.parseLong(t); ++ } ++ return 0; ++ } ++ + public void setDeliverTimeMs(long timeMs) { + this.putProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(timeMs)); + } + + public long getDeliverTimeMs() { +- return Long.parseLong(this.getUserProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS)); ++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS); ++ if (t != null) { ++ return Long.parseLong(t); ++ } ++ return 0; + } + } +-- +2.32.0.windows.2 + + +From 7e4879a3bc120d6289aabc8354a2811f349ac8a6 Mon Sep 17 00:00:00 2001 +From: fujian-zfj <2573259572@qq.com> +Date: Wed, 11 Oct 2023 14:45:07 +0800 +Subject: [PATCH 2/6] [ISSUE #7441] Fix log "Init the confirmOffset" keep + printing error in controller mode (#7442) + +* typo int readme[ecosystem] + +* fix keep printing log problem +--- + store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +index 456bf2b86..f98e9a284 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +@@ -580,7 +580,7 @@ public class CommitLog implements Swappable { + return this.defaultMessageStore.getMaxPhyOffset(); + } + // First time it will compute the confirmOffset. +- if (this.confirmOffset <= 0) { ++ if (this.confirmOffset < 0) { + setConfirmOffset(((AutoSwitchHAService) this.defaultMessageStore.getHaService()).computeConfirmOffset()); + log.info("Init the confirmOffset to {}.", this.confirmOffset); + } +-- +2.32.0.windows.2 + + +From 5d492c338258d07613103e6ae16df4c6fa5b3838 Mon Sep 17 00:00:00 2001 +From: rongtong +Date: Fri, 13 Oct 2023 11:23:30 +0800 +Subject: [PATCH 3/6] [ISSUE #7444] Fix testCalculateFileSizeInPath test can + not rerun in same environment (#7445) + +* Fix testCalculateFileSizeInPath test can not rerun in same environment + +* Ensure that files are always deleted +--- + .../apache/rocketmq/common/UtilAllTest.java | 83 +++++++++++-------- + 1 file changed, 48 insertions(+), 35 deletions(-) + +diff --git a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java +index f568a65f4..a0653d7fc 100644 +--- a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java ++++ b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java +@@ -238,41 +238,54 @@ public class UtilAllTest { + */ + String basePath = System.getProperty("java.io.tmpdir") + File.separator + "testCalculateFileSizeInPath"; + File baseFile = new File(basePath); +- // test empty path +- assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile)); +- +- // create baseDir +- assertTrue(baseFile.mkdirs()); +- +- File file0 = new File(baseFile, "file_0"); +- assertTrue(file0.createNewFile()); +- writeFixedBytesToFile(file0, 1313); +- +- assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile)); +- +- // build a file tree like above +- File dir1 = new File(baseFile, "dir_1"); +- dir1.mkdirs(); +- File file10 = new File(dir1, "file_1_0"); +- File file11 = new File(dir1, "file_1_1"); +- File dir12 = new File(dir1, "dir_1_2"); +- dir12.mkdirs(); +- File file120 = new File(dir12, "file_1_2_0"); +- File dir2 = new File(baseFile, "dir_2"); +- dir2.mkdirs(); +- +- // write all file with 1313 bytes data +- assertTrue(file10.createNewFile()); +- writeFixedBytesToFile(file10, 1313); +- assertTrue(file11.createNewFile()); +- writeFixedBytesToFile(file11, 1313); +- assertTrue(file120.createNewFile()); +- writeFixedBytesToFile(file120, 1313); +- +- assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile)); +- +- // clear all file +- baseFile.deleteOnExit(); ++ try { ++ // test empty path ++ assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile)); ++ ++ // create baseDir ++ assertTrue(baseFile.mkdirs()); ++ ++ File file0 = new File(baseFile, "file_0"); ++ assertTrue(file0.createNewFile()); ++ writeFixedBytesToFile(file0, 1313); ++ ++ assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile)); ++ ++ // build a file tree like above ++ File dir1 = new File(baseFile, "dir_1"); ++ dir1.mkdirs(); ++ File file10 = new File(dir1, "file_1_0"); ++ File file11 = new File(dir1, "file_1_1"); ++ File dir12 = new File(dir1, "dir_1_2"); ++ dir12.mkdirs(); ++ File file120 = new File(dir12, "file_1_2_0"); ++ File dir2 = new File(baseFile, "dir_2"); ++ dir2.mkdirs(); ++ ++ // write all file with 1313 bytes data ++ assertTrue(file10.createNewFile()); ++ writeFixedBytesToFile(file10, 1313); ++ assertTrue(file11.createNewFile()); ++ writeFixedBytesToFile(file11, 1313); ++ assertTrue(file120.createNewFile()); ++ writeFixedBytesToFile(file120, 1313); ++ ++ assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile)); ++ } finally { ++ deleteFolder(baseFile); ++ } ++ } ++ ++ public static void deleteFolder(File folder) { ++ if (folder.isDirectory()) { ++ File[] files = folder.listFiles(); ++ if (files != null) { ++ for (File file : files) { ++ deleteFolder(file); ++ } ++ } ++ } ++ folder.delete(); + } + + private void writeFixedBytesToFile(File file, int size) throws Exception { +-- +2.32.0.windows.2 + + +From 28427d40129e3aa0c6f951535617e5cac0a8211b Mon Sep 17 00:00:00 2001 +From: Lei Sun +Date: Fri, 13 Oct 2023 13:42:27 +0800 +Subject: [PATCH 4/6] [ISSUE #7425] Add RoccketmqControllerConsole log to fix + bug (#7458) + +--- + .../org/apache/rocketmq/common/constant/LoggerName.java | 1 + + .../org/apache/rocketmq/controller/ControllerStartup.java | 7 ++++--- + controller/src/main/resources/rmq.controller.logback.xml | 4 ++++ + 3 files changed, 9 insertions(+), 3 deletions(-) + +diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +index cb04b00b3..61310893f 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +@@ -21,6 +21,7 @@ public class LoggerName { + public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; + public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; + public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; ++ public static final String CONTROLLER_CONSOLE_NAME = "RocketmqControllerConsole"; + public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; + public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; + public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; +diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java +index 401720d05..9e96a704d 100644 +--- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java ++++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java +@@ -94,9 +94,10 @@ public class ControllerStartup { + } + + if (commandLine.hasOption('p')) { +- MixAll.printObjectProperties(null, controllerConfig); +- MixAll.printObjectProperties(null, nettyServerConfig); +- MixAll.printObjectProperties(null, nettyClientConfig); ++ Logger console = LoggerFactory.getLogger(LoggerName.CONTROLLER_CONSOLE_NAME); ++ MixAll.printObjectProperties(console, controllerConfig); ++ MixAll.printObjectProperties(console, nettyServerConfig); ++ MixAll.printObjectProperties(console, nettyClientConfig); + System.exit(0); + } + +diff --git a/controller/src/main/resources/rmq.controller.logback.xml b/controller/src/main/resources/rmq.controller.logback.xml +index bb158213a..18083e8f9 100644 +--- a/controller/src/main/resources/rmq.controller.logback.xml ++++ b/controller/src/main/resources/rmq.controller.logback.xml +@@ -116,6 +116,10 @@ + + + ++ ++ ++ ++ + + + +-- +2.32.0.windows.2 + + +From dc62d7f2e1ce4f99364599f8e23d65fd88eb1cd4 Mon Sep 17 00:00:00 2001 +From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> +Date: Fri, 13 Oct 2023 13:45:48 +0800 +Subject: [PATCH 5/6] [ISSUE #7451] Override toString for + TopicConfigAndQueueMapping + +--- + .../statictopic/TopicConfigAndQueueMapping.java | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java +index c937fec23..d13692735 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java +@@ -16,6 +16,7 @@ + */ + package org.apache.rocketmq.remoting.protocol.statictopic; + ++import org.apache.commons.lang3.StringUtils; + import org.apache.commons.lang3.builder.EqualsBuilder; + import org.apache.commons.lang3.builder.HashCodeBuilder; + import org.apache.rocketmq.common.TopicConfig; +@@ -60,4 +61,13 @@ public class TopicConfigAndQueueMapping extends TopicConfig { + .append(mappingDetail) + .toHashCode(); + } ++ ++ @Override ++ public String toString() { ++ String string = super.toString(); ++ if (StringUtils.isNotBlank(string)) { ++ string = string.substring(0, string.length() - 1) + ", mappingDetail=" + mappingDetail + "]"; ++ } ++ return string; ++ } + } +-- +2.32.0.windows.2 + + +From 2113fa371b9c2bf7c512f8ad234e51c616f1362c Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Fri, 13 Oct 2023 13:47:09 +0800 +Subject: [PATCH 6/6] [ISSUE #7453] Fix the problem in constructing the + GetMessageResult (#7456) + +* Fix the problem in constructing the GetMessageResult + +* Optimize the initialization size of GetMessageResult +--- + .../apache/rocketmq/broker/processor/PeekMessageProcessor.java | 3 +-- + .../apache/rocketmq/broker/processor/PopMessageProcessor.java | 3 +-- + 2 files changed, 2 insertions(+), 4 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java +index a8358c4ff..e1e0e13e5 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java +@@ -129,8 +129,7 @@ public class PeekMessageProcessor implements NettyRequestProcessor { + } + int randomQ = random.nextInt(100); + int reviveQid = randomQ % this.brokerController.getBrokerConfig().getReviveQueueNum(); +- int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg(); +- GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg); ++ GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums()); + boolean needRetry = randomQ % 5 == 0; + long popTime = System.currentTimeMillis(); + long restNum = 0; +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +index 441f7de08..0d9bdf143 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +@@ -347,8 +347,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { + reviveQid = (int) Math.abs(ckMessageNumber.getAndIncrement() % this.brokerController.getBrokerConfig().getReviveQueueNum()); + } + +- int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg(); +- GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg); ++ GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums()); + ExpressionMessageFilter finalMessageFilter = messageFilter; + StringBuilder finalOrderCountInfo = orderCountInfo; + +-- +2.32.0.windows.2 + diff --git a/pom.xml b/pom.xml index 4202d4095..a3f7c2270 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ 1.29.0-alpha 2.0.6 2.20.29 - 1.0.3 + 1.0.2 2.13.4.2 @@ -713,7 +713,7 @@ ${slf4j-api.version} - io.github.aliyunmq + org.apache.rocketmq rocketmq-rocksdb ${rocksdb.version} diff --git a/store/pom.xml b/store/pom.xml index e979030e8..e1e616123 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -58,6 +58,10 @@ com.google.guava guava + + commons-io + commons-io + org.slf4j diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index f98e9a284..93102799b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -60,6 +60,8 @@ import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.util.LibC; +import org.rocksdb.RocksDBException; + import sun.nio.ch.DirectBuffer; /** @@ -299,8 +301,9 @@ public class CommitLog implements Swappable { /** * When the normal exit, data recovery, all memory data have been flush + * @throws RocksDBException only in rocksdb mode */ - public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { + public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable(); final List mappedFiles = this.mappedFileQueue.getMappedFiles(); @@ -369,21 +372,22 @@ public class CommitLog implements Swappable { this.setConfirmOffset(lastValidMsgPhyOffset); } - this.mappedFileQueue.setFlushedWhere(processOffset); - this.mappedFileQueue.setCommittedWhere(processOffset); - this.mappedFileQueue.truncateDirtyFiles(processOffset); - // Clear ConsumeQueue redundant data if (maxPhyOffsetOfConsumeQueue >= processOffset) { log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset); this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); } + + this.mappedFileQueue.setFlushedWhere(processOffset); + this.mappedFileQueue.setCommittedWhere(processOffset); + this.mappedFileQueue.truncateDirtyFiles(processOffset); } else { // Commitlog case files are deleted log.warn("The commitlog files are deleted, and delete the consume queue files"); this.mappedFileQueue.setFlushedWhere(0); this.mappedFileQueue.setCommittedWhere(0); - this.defaultMessageStore.destroyLogics(); + this.defaultMessageStore.getQueueStore().destroy(); + this.defaultMessageStore.getQueueStore().loadAfterDestroy(); } } @@ -626,8 +630,10 @@ public class CommitLog implements Swappable { return -1; } - @Deprecated - public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { + /** + * @throws RocksDBException only in rocksdb mode + */ + public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { // recover by the minimum time stamp boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable(); @@ -705,6 +711,9 @@ public class CommitLog implements Swappable { } } + // only for rocksdb mode + this.getMessageStore().finishCommitLogDispatch(); + processOffset += mappedFileOffset; if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) { if (this.defaultMessageStore.getConfirmOffset() < this.defaultMessageStore.getMinPhyOffset()) { @@ -717,22 +726,24 @@ public class CommitLog implements Swappable { } else { this.setConfirmOffset(lastValidMsgPhyOffset); } - this.mappedFileQueue.setFlushedWhere(processOffset); - this.mappedFileQueue.setCommittedWhere(processOffset); - this.mappedFileQueue.truncateDirtyFiles(processOffset); // Clear ConsumeQueue redundant data if (maxPhyOffsetOfConsumeQueue >= processOffset) { log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset); this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); } + + this.mappedFileQueue.setFlushedWhere(processOffset); + this.mappedFileQueue.setCommittedWhere(processOffset); + this.mappedFileQueue.truncateDirtyFiles(processOffset); } // Commitlog case files are deleted else { log.warn("The commitlog files are deleted, and delete the consume queue files"); this.mappedFileQueue.setFlushedWhere(0); this.mappedFileQueue.setCommittedWhere(0); - this.defaultMessageStore.destroyLogics(); + this.defaultMessageStore.getQueueStore().destroy(); + this.defaultMessageStore.getQueueStore().loadAfterDestroy(); } } @@ -755,7 +766,7 @@ public class CommitLog implements Swappable { this.getMessageStore().onCommitLogAppend(msg, result, commitLogFile); } - private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) { + private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) throws RocksDBException { ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); int magicCode = byteBuffer.getInt(MessageDecoder.MESSAGE_MAGIC_CODE_POSITION); @@ -763,28 +774,37 @@ public class CommitLog implements Swappable { return false; } - int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION); - int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; - int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornhostLength; - long storeTimestamp = byteBuffer.getLong(msgStoreTimePos); - if (0 == storeTimestamp) { - return false; - } - - if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() - && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { - if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { - log.info("find check timestamp, {} {}", - storeTimestamp, - UtilAll.timeMillisToHumanString(storeTimestamp)); + if (this.defaultMessageStore.getMessageStoreConfig().isEnableRocksDBStore()) { + final long maxPhyOffsetInConsumeQueue = this.defaultMessageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue(); + long phyOffset = byteBuffer.getLong(MessageDecoder.MESSAGE_PHYSIC_OFFSET_POSITION); + if (phyOffset <= maxPhyOffsetInConsumeQueue) { + log.info("find check. beginPhyOffset: {}, maxPhyOffsetInConsumeQueue: {}", phyOffset, maxPhyOffsetInConsumeQueue); return true; } } else { - if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { - log.info("find check timestamp, {} {}", - storeTimestamp, - UtilAll.timeMillisToHumanString(storeTimestamp)); - return true; + int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION); + int bornHostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; + int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornHostLength; + long storeTimestamp = byteBuffer.getLong(msgStoreTimePos); + if (0 == storeTimestamp) { + return false; + } + + if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() + && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { + if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { + log.info("find check timestamp, {} {}", + storeTimestamp, + UtilAll.timeMillisToHumanString(storeTimestamp)); + return true; + } + } else { + if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { + log.info("find check timestamp, {} {}", + storeTimestamp, + UtilAll.timeMillisToHumanString(storeTimestamp)); + return true; + } } } @@ -958,8 +978,6 @@ public class CommitLog implements Swappable { beginTimeInLock = 0; return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result)); case UNKNOWN_ERROR: - beginTimeInLock = 0; - return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); default: beginTimeInLock = 0; return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); @@ -974,6 +992,8 @@ public class CommitLog implements Swappable { if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) { this.defaultMessageStore.increaseOffset(msg, getMessageNum(msg)); } + } catch (RocksDBException e) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); } finally { topicQueueLock.unlock(topicQueueKey); } @@ -997,7 +1017,7 @@ public class CommitLog implements Swappable { public CompletableFuture asyncPutMessages(final MessageExtBatch messageExtBatch) { messageExtBatch.setStoreTimestamp(System.currentTimeMillis()); - AppendMessageResult result; + AppendMessageResult result = null; StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); @@ -1133,7 +1153,9 @@ public class CommitLog implements Swappable { if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) { this.defaultMessageStore.increaseOffset(messageExtBatch, (short) putMessageContext.getBatchSize()); } - } finally { + } catch (RocksDBException e) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); + } finally { topicQueueLock.unlock(topicQueueKey); } diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java index 9d6fa6ad9..f3a7b7c5c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java @@ -17,6 +17,8 @@ package org.apache.rocketmq.store; +import org.rocksdb.RocksDBException; + /** * Dispatcher of commit log. */ @@ -25,6 +27,7 @@ public interface CommitLogDispatcher { /** * Dispatch messages from store to build consume queues, indexes, and filter data * @param request dispatch message request + * @throws RocksDBException only in rocksdb mode */ - void dispatch(final DispatchRequest request); + void dispatch(final DispatchRequest request) throws RocksDBException; } diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index 56bee2af3..623509c8b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -24,13 +24,12 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.attribute.CQType; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.config.BrokerRole; @@ -39,9 +38,9 @@ import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.FileQueueLifeCycle; +import org.apache.rocketmq.store.queue.MultiDispatch; import org.apache.rocketmq.store.queue.QueueOffsetOperator; import org.apache.rocketmq.store.queue.ReferredIterator; -import org.apache.rocketmq.store.timer.TimerMessageStore; public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); @@ -703,7 +702,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp()); } this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(request.getStoreTimestamp()); - if (checkMultiDispatchQueue(request)) { + if (MultiDispatch.checkMultiDispatchQueue(this.messageStore.getMessageStoreConfig(), request)) { multiDispatchLmqQueue(request, maxRetries); } return; @@ -725,25 +724,6 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { this.messageStore.getRunningFlags().makeLogicsQueueError(); } - private boolean checkMultiDispatchQueue(DispatchRequest dispatchRequest) { - if (!this.messageStore.getMessageStoreConfig().isEnableMultiDispatch() - || dispatchRequest.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) - || dispatchRequest.getTopic().equals(TimerMessageStore.TIMER_TOPIC) - || dispatchRequest.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC)) { - return false; - } - Map prop = dispatchRequest.getPropertiesMap(); - if (prop == null || prop.isEmpty()) { - return false; - } - String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); - String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); - if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { - return false; - } - return true; - } - private void multiDispatchLmqQueue(DispatchRequest request, int maxRetries) { Map prop = request.getPropertiesMap(); String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); @@ -765,9 +745,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { queueId = 0; } doDispatchLmqQueue(request, maxRetries, queueName, queueOffset, queueId); - } - return; } private void doDispatchLmqQueue(DispatchRequest request, int maxRetries, String queueName, long queueOffset, @@ -802,7 +780,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. - if (!isNeedHandleMultiDispatch(msg)) { + if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { return; } String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); @@ -812,14 +790,14 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); Long[] queueOffsets = new Long[queues.length]; for (int i = 0; i < queues.length; i++) { - String key = queueKey(queues[i], msg); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { + String key = MultiDispatch.lmqQueueKey(queues[i]); queueOffsets[i] = queueOffsetOperator.getLmqOffset(key); } } MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); - removeWaitStorePropertyString(msg); + msg.removeWaitStorePropertyString(); } @Override @@ -830,7 +808,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. - if (!isNeedHandleMultiDispatch(msg)) { + if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { return; } String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); @@ -839,45 +817,13 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { } String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); for (int i = 0; i < queues.length; i++) { - String key = queueKey(queues[i], msg); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { + String key = MultiDispatch.lmqQueueKey(queues[i]); queueOffsetOperator.increaseLmqOffset(key, (short) 1); } } } - public boolean isNeedHandleMultiDispatch(MessageExtBrokerInner msg) { - return messageStore.getMessageStoreConfig().isEnableMultiDispatch() - && !msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) - && !msg.getTopic().equals(TimerMessageStore.TIMER_TOPIC) - && !msg.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); - } - - public String queueKey(String queueName, MessageExtBrokerInner msgInner) { - StringBuilder keyBuilder = new StringBuilder(); - keyBuilder.append(queueName); - keyBuilder.append('-'); - int queueId = msgInner.getQueueId(); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { - queueId = 0; - } - keyBuilder.append(queueId); - return keyBuilder.toString(); - } - - private void removeWaitStorePropertyString(MessageExtBrokerInner msgInner) { - if (msgInner.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) { - // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message. - // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it. - String waitStoreMsgOKValue = msgInner.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); - msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later - msgInner.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue); - } else { - msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - } - } - private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, final long cqOffset) { @@ -965,6 +911,11 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { return new ConsumeQueueIterator(sbr); } + @Override + public ReferredIterator iterateFrom(long startIndex, int count) { + return iterateFrom(startIndex); + } + @Override public CqUnit get(long offset) { ReferredIterator it = iterateFrom(offset); @@ -974,6 +925,20 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { return it.nextAndRelease(); } + @Override + public Pair getCqUnitAndStoreTime(long index) { + CqUnit cqUnit = get(index); + Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); + return new Pair<>(cqUnit, messageStoreTime); + } + + @Override + public Pair getEarliestUnitAndStoreTime() { + CqUnit cqUnit = getEarliestUnit(); + Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); + return new Pair<>(cqUnit, messageStoreTime); + } + @Override public CqUnit getEarliestUnit() { /** diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 02ea47f13..99a54e2d7 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -105,32 +105,35 @@ import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.ConsumeQueueStore; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.util.PerfCounter; +import org.rocksdb.RocksDBException; public class DefaultMessageStore implements MessageStore { - private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + protected static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); public final PerfCounter.Ticks perfs = new PerfCounter.Ticks(LOGGER); private final MessageStoreConfig messageStoreConfig; // CommitLog - private final CommitLog commitLog; + protected final CommitLog commitLog; - private final ConsumeQueueStore consumeQueueStore; + protected final ConsumeQueueStoreInterface consumeQueueStore; private final FlushConsumeQueueService flushConsumeQueueService; - private final CleanCommitLogService cleanCommitLogService; + protected final CleanCommitLogService cleanCommitLogService; private final CleanConsumeQueueService cleanConsumeQueueService; private final CorrectLogicOffsetService correctLogicOffsetService; - private final IndexService indexService; + protected final IndexService indexService; private final AllocateMappedFileService allocateMappedFileService; @@ -147,7 +150,7 @@ public class DefaultMessageStore implements MessageStore { private final TransientStorePool transientStorePool; - private final RunningFlags runningFlags = new RunningFlags(); + protected final RunningFlags runningFlags = new RunningFlags(); private final SystemClock systemClock = new SystemClock(); private final ScheduledExecutorService scheduledExecutorService; @@ -156,6 +159,7 @@ public class DefaultMessageStore implements MessageStore { private final BrokerConfig brokerConfig; private volatile boolean shutdown = true; + protected boolean notifyMessageArriveInBatch = false; private StoreCheckpoint storeCheckpoint; private TimerMessageStore timerMessageStore; @@ -182,7 +186,7 @@ public class DefaultMessageStore implements MessageStore { private volatile long brokerInitMaxOffset = -1L; - protected List putMessageHookList = new ArrayList<>(); + private List putMessageHookList = new ArrayList<>(); private SendMessageBackHook sendMessageBackHook; @@ -222,12 +226,12 @@ public class DefaultMessageStore implements MessageStore { this.commitLog = new CommitLog(this); } - this.consumeQueueStore = new ConsumeQueueStore(this, this.messageStoreConfig); + this.consumeQueueStore = createConsumeQueueStore(); - this.flushConsumeQueueService = new FlushConsumeQueueService(); + this.flushConsumeQueueService = createFlushConsumeQueueService(); this.cleanCommitLogService = new CleanCommitLogService(); - this.cleanConsumeQueueService = new CleanConsumeQueueService(); - this.correctLogicOffsetService = new CorrectLogicOffsetService(); + this.cleanConsumeQueueService = createCleanConsumeQueueService(); + this.correctLogicOffsetService = createCorrectLogicOffsetService(); this.storeStatsService = new StoreStatsService(getBrokerIdentity()); this.indexService = new IndexService(this); @@ -273,6 +277,22 @@ public class DefaultMessageStore implements MessageStore { parseDelayLevel(); } + public ConsumeQueueStoreInterface createConsumeQueueStore() { + return new ConsumeQueueStore(this); + } + + public CleanConsumeQueueService createCleanConsumeQueueService() { + return new CleanConsumeQueueService(); + } + + public FlushConsumeQueueService createFlushConsumeQueueService() { + return new FlushConsumeQueueService(); + } + + public CorrectLogicOffsetService createCorrectLogicOffsetService() { + return new CorrectLogicOffsetService(); + } + public boolean parseDelayLevel() { HashMap timeUnitTable = new HashMap<>(); timeUnitTable.put("s", 1000L); @@ -305,7 +325,7 @@ public class DefaultMessageStore implements MessageStore { } @Override - public void truncateDirtyLogicFiles(long phyOffset) { + public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException { this.consumeQueueStore.truncateDirty(phyOffset); } @@ -393,6 +413,7 @@ public class DefaultMessageStore implements MessageStore { this.flushConsumeQueueService.start(); this.commitLog.start(); + this.consumeQueueStore.start(); this.storeStatsService.start(); if (this.haService != null) { @@ -481,6 +502,7 @@ public class DefaultMessageStore implements MessageStore { this.storeStatsService.shutdown(); this.commitLog.shutdown(); this.reputMessageService.shutdown(); + this.consumeQueueStore.shutdown(); // dispatch-related services must be shut down after reputMessageService this.indexService.shutdown(); if (this.compactionService != null) { @@ -515,7 +537,7 @@ public class DefaultMessageStore implements MessageStore { @Override public void destroy() { - this.destroyLogics(); + this.consumeQueueStore.destroy(); this.commitLog.destroy(); this.indexService.destroy(); this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); @@ -541,11 +563,6 @@ public class DefaultMessageStore implements MessageStore { return commitLogSize + consumeQueueSize + indexFileSize; } - @Override - public void destroyLogics() { - this.consumeQueueStore.destroy(); - } - @Override public CompletableFuture asyncPutMessage(MessageExtBrokerInner msg) { @@ -687,7 +704,7 @@ public class DefaultMessageStore implements MessageStore { return commitLog; } - public void truncateDirtyFiles(long offsetToTruncate) { + public void truncateDirtyFiles(long offsetToTruncate) throws RocksDBException { LOGGER.info("truncate dirty files to {}", offsetToTruncate); @@ -700,12 +717,12 @@ public class DefaultMessageStore implements MessageStore { long oldReputFromOffset = this.reputMessageService.getReputFromOffset(); - // truncate commitLog - this.commitLog.truncateDirtyFiles(offsetToTruncate); - // truncate consume queue this.truncateDirtyLogicFiles(offsetToTruncate); + // truncate commitLog + this.commitLog.truncateDirtyFiles(offsetToTruncate); + this.recoverTopicQueueTable(); if (!messageStoreConfig.isEnableBuildConsumeQueueConcurrently()) { @@ -723,7 +740,7 @@ public class DefaultMessageStore implements MessageStore { } @Override - public boolean truncateFiles(long offsetToTruncate) { + public boolean truncateFiles(long offsetToTruncate) throws RocksDBException { if (offsetToTruncate >= this.getMaxPhyOffset()) { LOGGER.info("no need to truncate files, truncate offset is {}, max physical offset is {}", offsetToTruncate, this.getMaxPhyOffset()); return true; @@ -825,17 +842,19 @@ public class DefaultMessageStore implements MessageStore { while (getResult.getBufferTotalSize() <= 0 && nextBeginOffset < maxOffset && cqFileNum++ < this.messageStoreConfig.getTravelCqFileNumWhenGetMessage()) { - ReferredIterator bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset); - - if (bufferConsumeQueue == null) { - status = GetMessageStatus.OFFSET_FOUND_NULL; - nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset)); - LOGGER.warn("consumer request topic: " + topic + "offset: " + offset + " minOffset: " + minOffset + " maxOffset: " - + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset); - break; - } + ReferredIterator bufferConsumeQueue = null; try { + bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset, maxMsgNums); + + if (bufferConsumeQueue == null) { + status = GetMessageStatus.OFFSET_FOUND_NULL; + nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset)); + LOGGER.warn("consumer request topic: " + topic + ", offset: " + offset + ", minOffset: " + minOffset + ", maxOffset: " + + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset); + break; + } + long nextPhyFileStartOffset = Long.MIN_VALUE; while (bufferConsumeQueue.hasNext() && nextBeginOffset < maxOffset) { @@ -905,8 +924,13 @@ public class DefaultMessageStore implements MessageStore { status = GetMessageStatus.FOUND; nextPhyFileStartOffset = Long.MIN_VALUE; } + } catch (RocksDBException e) { + ERROR_LOG.error("getMessage Failed. cid: {}, topic: {}, queueId: {}, offset: {}, minOffset: {}, maxOffset: {}, {}", + group, topic, queueId, offset, minOffset, maxOffset, e.getMessage()); } finally { - bufferConsumeQueue.release(); + if (bufferConsumeQueue != null) { + bufferConsumeQueue.release(); + } } } @@ -975,12 +999,12 @@ public class DefaultMessageStore implements MessageStore { @Override public long getMinOffsetInQueue(String topic, int queueId) { - ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); - if (logic != null) { - return logic.getMinOffsetInQueue(); + try { + return this.consumeQueueStore.getMinOffsetInQueue(topic, queueId); + } catch (RocksDBException e) { + ERROR_LOG.error("getMinOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e); + return -1; } - - return -1; } @Override @@ -997,38 +1021,27 @@ public class DefaultMessageStore implements MessageStore { public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) { ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); if (consumeQueue != null) { - - ReferredIterator bufferConsumeQueue = consumeQueue.iterateFrom(consumeQueueOffset); - if (bufferConsumeQueue != null) { - try { - if (bufferConsumeQueue.hasNext()) { - long offsetPy = bufferConsumeQueue.next().getPos(); - return offsetPy; - } - } finally { - bufferConsumeQueue.release(); - } + CqUnit cqUnit = consumeQueue.get(consumeQueueOffset); + if (cqUnit != null) { + return cqUnit.getPos(); } } - return 0; } @Override public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) { - return getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER); + return this.getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER); } + @Override public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) { - ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); - if (logic != null) { - long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType); - // Make sure the result offset is in valid range. - resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue()); - resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue()); - return resultOffset; + try { + return this.consumeQueueStore.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType); + } catch (RocksDBException e) { + ERROR_LOG.error("getOffsetInQueueByTime Failed. topic: {}, queueId: {}, timestamp: {} boundaryType: {}, {}", + topic, queueId, timestamp, boundaryType, e.getMessage()); } - return 0; } @@ -1088,6 +1101,10 @@ public class DefaultMessageStore implements MessageStore { return StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()); } + public MessageArrivingListener getMessageArrivingListener() { + return messageArrivingListener; + } + @Override public HashMap getRuntimeInfo() { HashMap result = this.storeStatsService.getRuntimeInfo(); @@ -1121,7 +1138,6 @@ public class DefaultMessageStore implements MessageStore { return this.commitLog.getMaxOffset(); } - @Override public long getMinPhyOffset() { return this.commitLog.getMinOffset(); @@ -1141,7 +1157,10 @@ public class DefaultMessageStore implements MessageStore { public long getEarliestMessageTime(String topic, int queueId) { ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); if (logicQueue != null) { - return getStoreTime(logicQueue.getEarliestUnit()); + Pair pair = logicQueue.getEarliestUnitAndStoreTime(); + if (pair != null && pair.getObject2() != null) { + return pair.getObject2(); + } } return -1; @@ -1152,19 +1171,6 @@ public class DefaultMessageStore implements MessageStore { return CompletableFuture.completedFuture(getEarliestMessageTime(topic, queueId)); } - protected long getStoreTime(CqUnit result) { - if (result != null) { - try { - final long phyOffset = result.getPos(); - final int size = result.getSize(); - long storeTime = this.getCommitLog().pickupStoreTimestamp(phyOffset, size); - return storeTime; - } catch (Exception e) { - } - } - return -1; - } - @Override public long getEarliestMessageTime() { long minPhyOffset = this.getMinPhyOffset(); @@ -1179,13 +1185,16 @@ public class DefaultMessageStore implements MessageStore { public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) { ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); if (logicQueue != null) { - return getStoreTime(logicQueue.get(consumeQueueOffset)); + Pair pair = logicQueue.getCqUnitAndStoreTime(consumeQueueOffset); + if (pair != null && pair.getObject2() != null) { + return pair.getObject2(); + } } - return -1; } - @Override public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, + @Override + public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, long consumeQueueOffset) { return CompletableFuture.completedFuture(getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset)); } @@ -1354,6 +1363,7 @@ public class DefaultMessageStore implements MessageStore { * If offset table is cleaned, and old messages are dispatching after the old consume queue is cleaned, * consume queue will be created with old offset, then later message with new offset table can not be * dispatched to consume queue. + * @throws RocksDBException only in rocksdb mode */ @Override public int deleteTopics(final Set deleteTopics) { @@ -1363,17 +1373,19 @@ public class DefaultMessageStore implements MessageStore { int deleteCount = 0; for (String topic : deleteTopics) { - ConcurrentMap queueTable = - this.consumeQueueStore.getConsumeQueueTable().get(topic); + ConcurrentMap queueTable = this.consumeQueueStore.findConsumeQueueMap(topic); if (queueTable == null || queueTable.isEmpty()) { continue; } for (ConsumeQueueInterface cq : queueTable.values()) { - this.consumeQueueStore.destroy(cq); - LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}", - cq.getTopic(), cq.getQueueId()); + try { + this.consumeQueueStore.destroy(cq); + } catch (RocksDBException e) { + LOGGER.error("DeleteTopic: ConsumeQueue cleans error!, topic={}, queueId={}", cq.getTopic(), cq.getQueueId(), e); + } + LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}", cq.getTopic(), cq.getQueueId()); this.consumeQueueStore.removeTopicQueueTable(cq.getTopic(), cq.getQueueId()); } @@ -1852,14 +1864,18 @@ public class DefaultMessageStore implements MessageStore { return file.exists(); } - private void recover(final boolean lastExitOK) { - boolean recoverConcurrently = this.brokerConfig.isRecoverConcurrently(); + private boolean isRecoverConcurrently() { + return this.brokerConfig.isRecoverConcurrently() && !this.messageStoreConfig.isEnableRocksDBStore(); + } + + private void recover(final boolean lastExitOK) throws RocksDBException { + boolean recoverConcurrently = this.isRecoverConcurrently(); LOGGER.info("message store recover mode: {}", recoverConcurrently ? "concurrent" : "normal"); // recover consume queue long recoverConsumeQueueStart = System.currentTimeMillis(); this.recoverConsumeQueue(); - long maxPhyOffsetOfConsumeQueue = this.getMaxOffsetInConsumeQueue(); + long maxPhyOffsetOfConsumeQueue = this.consumeQueueStore.getMaxPhyOffsetInConsumeQueue(); long recoverConsumeQueueEnd = System.currentTimeMillis(); // recover commitlog @@ -1894,23 +1910,25 @@ public class DefaultMessageStore implements MessageStore { return messageStoreConfig; } + @Override + public void finishCommitLogDispatch() { + // ignore + } + @Override public TransientStorePool getTransientStorePool() { return transientStorePool; } private void recoverConsumeQueue() { - if (!this.brokerConfig.isRecoverConcurrently()) { + if (!this.isRecoverConcurrently()) { this.consumeQueueStore.recover(); } else { this.consumeQueueStore.recoverConcurrently(); } } - private long getMaxOffsetInConsumeQueue() { - return this.consumeQueueStore.getMaxOffsetInConsumeQueue(); - } - + @Override public void recoverTopicQueueTable() { long minPhyOffset = this.commitLog.getMinOffset(); this.consumeQueueStore.recoverOffsetTable(minPhyOffset); @@ -1949,13 +1967,17 @@ public class DefaultMessageStore implements MessageStore { return runningFlags; } - public void doDispatch(DispatchRequest req) { + public void doDispatch(DispatchRequest req) throws RocksDBException { for (CommitLogDispatcher dispatcher : this.dispatcherList) { dispatcher.dispatch(req); } } - public void putMessagePositionInfo(DispatchRequest dispatchRequest) { + /** + * @param dispatchRequest + * @throws RocksDBException only in rocksdb mode + */ + protected void putMessagePositionInfo(DispatchRequest dispatchRequest) throws RocksDBException { this.consumeQueueStore.putMessagePositionInfoWrapper(dispatchRequest); } @@ -2054,7 +2076,7 @@ public class DefaultMessageStore implements MessageStore { } @Override - public ConsumeQueueStore getQueueStore() { + public ConsumeQueueStoreInterface getQueueStore() { return consumeQueueStore; } @@ -2065,7 +2087,7 @@ public class DefaultMessageStore implements MessageStore { @Override public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, - boolean isRecover, boolean isFileEnd) { + boolean isRecover, boolean isFileEnd) throws RocksDBException { if (doDispatch && !isFileEnd) { this.doDispatch(dispatchRequest); } @@ -2082,7 +2104,7 @@ public class DefaultMessageStore implements MessageStore { } @Override - public void assignOffset(MessageExtBrokerInner msg) { + public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException { final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { @@ -2127,12 +2149,12 @@ public class DefaultMessageStore implements MessageStore { class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher { @Override - public void dispatch(DispatchRequest request) { + public void dispatch(DispatchRequest request) throws RocksDBException { final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag()); switch (tranType) { case MessageSysFlag.TRANSACTION_NOT_TYPE: case MessageSysFlag.TRANSACTION_COMMIT_TYPE: - DefaultMessageStore.this.putMessagePositionInfo(request); + putMessagePositionInfo(request); break; case MessageSysFlag.TRANSACTION_PREPARED_TYPE: case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: @@ -2278,7 +2300,7 @@ public class DefaultMessageStore implements MessageStore { return DefaultMessageStore.this.brokerConfig.getIdentifier() + CleanCommitLogService.class.getSimpleName(); } - private boolean isTimeToDelete() { + protected boolean isTimeToDelete() { String when = DefaultMessageStore.this.getMessageStoreConfig().getDeleteWhen(); if (UtilAll.isItTimeToDo(when)) { DefaultMessageStore.LOGGER.info("it's time to reclaim disk space, " + when); @@ -2436,7 +2458,7 @@ public class DefaultMessageStore implements MessageStore { } class CleanConsumeQueueService { - private long lastPhysicalMinOffset = 0; + protected long lastPhysicalMinOffset = 0; public void run() { try { @@ -2446,7 +2468,7 @@ public class DefaultMessageStore implements MessageStore { } } - private void deleteExpiredFiles() { + protected void deleteExpiredFiles() { int deleteLogicsFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval(); long minOffset = DefaultMessageStore.this.commitLog.getMinOffset(); @@ -2551,7 +2573,7 @@ public class DefaultMessageStore implements MessageStore { if (cqUnit.getPos() >= minPhyOffset) { - // Normal case, do not need correct. + // Normal case, do not need to correct. return false; } } @@ -2741,6 +2763,18 @@ public class DefaultMessageStore implements MessageStore { } + @Override + public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { + if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() + && DefaultMessageStore.this.messageArrivingListener != null) { + DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), + dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, + dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), + dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); + DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest); + } + } + class ReputMessageService extends ServiceThread { protected volatile long reputFromOffset = 0; @@ -2810,13 +2844,8 @@ public class DefaultMessageStore implements MessageStore { if (size > 0) { DefaultMessageStore.this.doDispatch(dispatchRequest); - if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() - && DefaultMessageStore.this.messageArrivingListener != null) { - DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), - dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, - dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), - dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); - notifyMessageArrive4MultiQueue(dispatchRequest); + if (!notifyMessageArriveInBatch) { + notifyMessageArriveIfNecessary(dispatchRequest); } this.reputFromOffset += size; @@ -2850,9 +2879,14 @@ public class DefaultMessageStore implements MessageStore { } } } + } catch (RocksDBException e) { + ERROR_LOG.info("dispatch message to cq exception. reputFromOffset: {}", this.reputFromOffset, e); + return; } finally { result.release(); } + + finishCommitLogDispatch(); } } @@ -2989,7 +3023,7 @@ public class DefaultMessageStore implements MessageStore { // dispatchRequestsList:[ // {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}, // {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}] - private void dispatch() { + private void dispatch() throws Exception { dispatchRequestsList.clear(); dispatchRequestOrderlyQueue.get(dispatchRequestsList); if (!dispatchRequestsList.isEmpty()) { @@ -2997,21 +3031,15 @@ public class DefaultMessageStore implements MessageStore { for (DispatchRequest dispatchRequest : dispatchRequests) { DefaultMessageStore.this.doDispatch(dispatchRequest); // wake up long-polling - if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() - && DefaultMessageStore.this.messageArrivingListener != null) { - DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), - dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, - dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), - dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); - DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest); - } + DefaultMessageStore.this.notifyMessageArriveIfNecessary(dispatchRequest); + if (!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() && - DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) { + DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) { DefaultMessageStore.this.storeStatsService - .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1); + .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1); DefaultMessageStore.this.storeStatsService - .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()) - .add(dispatchRequest.getMsgSize()); + .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()) + .add(dispatchRequest.getMsgSize()); } } } @@ -3079,7 +3107,7 @@ public class DefaultMessageStore implements MessageStore { public void doReput() { if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) { LOGGER.warn("The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.", - this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); + this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset(); } for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) { @@ -3138,6 +3166,9 @@ public class DefaultMessageStore implements MessageStore { result.release(); } } + + // only for rocksdb mode + finishCommitLogDispatch(); } /** @@ -3180,8 +3211,8 @@ public class DefaultMessageStore implements MessageStore { if (this.isCommitLogAvailable()) { LOGGER.warn("shutdown concurrentReputMessageService, but CommitLog have not finish to be dispatched, CommitLog max" + - " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(), - this.reputFromOffset); + " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(), + this.reputFromOffset); } this.mainBatchDispatchRequestService.shutdown(); diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java index 989cbbe31..814c6d1bf 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java @@ -16,10 +16,6 @@ */ package org.apache.rocketmq.store; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.sdk.metrics.InstrumentSelector; -import io.opentelemetry.sdk.metrics.ViewBuilder; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.LinkedList; @@ -40,10 +36,15 @@ import org.apache.rocketmq.store.hook.PutMessageHook; import org.apache.rocketmq.store.hook.SendMessageBackHook; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; -import org.apache.rocketmq.store.queue.ConsumeQueueStore; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.util.PerfCounter; +import org.rocksdb.RocksDBException; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.ViewBuilder; /** * This class defines contracting interfaces to implement, allowing third-party vendor to use customized message store. @@ -545,7 +546,7 @@ public interface MessageStore { void setConfirmOffset(long phyOffset); /** - * Check if the operation system page cache is busy or not. + * Check if the operating system page cache is busy or not. * * @return true if the OS page cache is busy; false otherwise. */ @@ -620,9 +621,18 @@ public interface MessageStore { * @param commitLogFile commit log file * @param isRecover is from recover process * @param isFileEnd if the dispatch request represents 'file end' + * @throws RocksDBException only in rocksdb mode */ void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, - boolean isRecover, boolean isFileEnd); + boolean isRecover, boolean isFileEnd) throws RocksDBException; + + /** + * Only used in rocksdb mode, because we build consumeQueue in batch(default 16 dispatchRequests) + * It will be triggered in two cases: + * @see org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput + * @see CommitLog#recoverAbnormally + */ + void finishCommitLogDispatch(); /** * Get the message store config @@ -691,13 +701,9 @@ public interface MessageStore { * Truncate dirty logic files * * @param phyOffset physical offset + * @throws RocksDBException only in rocksdb mode */ - void truncateDirtyLogicFiles(long phyOffset); - - /** - * Destroy logics files - */ - void destroyLogics(); + void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException; /** * Unlock mappedFile @@ -718,7 +724,7 @@ public interface MessageStore { * * @return the queue store */ - ConsumeQueueStore getQueueStore(); + ConsumeQueueStoreInterface getQueueStore(); /** * If 'sync disk flush' is configured in this message store @@ -739,8 +745,9 @@ public interface MessageStore { * yourself. * * @param msg message + * @throws RocksDBException */ - void assignOffset(MessageExtBrokerInner msg); + void assignOffset(MessageExtBrokerInner msg) throws RocksDBException; /** * Increase queue offset in memory table. If there is a race condition, you need to lock/unlock this method @@ -835,14 +842,15 @@ public interface MessageStore { * * @param offsetToTruncate offset to truncate * @return true if truncate succeed, false otherwise + * @throws RocksDBException only in rocksdb mode */ - boolean truncateFiles(long offsetToTruncate); + boolean truncateFiles(long offsetToTruncate) throws RocksDBException; /** - * Check if the offset is align with one message. + * Check if the offset is aligned with one message. * * @param offset offset to check - * @return true if align, false otherwise + * @return true if aligned, false otherwise */ boolean isOffsetAligned(long offset); @@ -971,4 +979,14 @@ public interface MessageStore { * @param attributesBuilderSupplier metrics attributes builder */ void initMetrics(Meter meter, Supplier attributesBuilderSupplier); + + /** + * Recover topic queue table + */ + void recoverTopicQueueTable(); + + /** + * notify message arrive if necessary + */ + void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest); } diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java new file mode 100644 index 000000000..87ccb5474 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; +import org.apache.rocketmq.store.queue.RocksDBConsumeQueue; +import org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.rocksdb.RocksDBException; + +public class RocksDBMessageStore extends DefaultMessageStore { + + public RocksDBMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, + final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws + IOException { + super(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, topicConfigTable); + notifyMessageArriveInBatch = true; + } + + @Override + public ConsumeQueueStoreInterface createConsumeQueueStore() { + return new RocksDBConsumeQueueStore(this); + } + + @Override + public CleanConsumeQueueService createCleanConsumeQueueService() { + return new RocksDBCleanConsumeQueueService(); + } + + @Override + public FlushConsumeQueueService createFlushConsumeQueueService() { + return new RocksDBFlushConsumeQueueService(); + } + + @Override + public CorrectLogicOffsetService createCorrectLogicOffsetService() { + return new RocksDBCorrectLogicOffsetService(); + } + + /** + * Try to set topicQueueTable = new HashMap<>(), otherwise it will cause bug when broker role changes. + * And unlike method in DefaultMessageStore, we don't need to really recover topic queue table advance, + * because we can recover topic queue table from rocksdb when we need to use it. + * @see RocksDBConsumeQueue#assignQueueOffset + */ + @Override + public void recoverTopicQueueTable() { + this.consumeQueueStore.setTopicQueueTable(new ConcurrentHashMap<>()); + } + + @Override + public void finishCommitLogDispatch() { + try { + putMessagePositionInfo(null); + } catch (RocksDBException e) { + ERROR_LOG.info("try to finish commitlog dispatch error.", e); + } + } + + @Override + public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) { + return findConsumeQueue(topic, queueId); + } + + class RocksDBCleanConsumeQueueService extends CleanConsumeQueueService { + private final double diskSpaceWarningLevelRatio = + Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceWarningLevelRatio", "0.90")); + + private final double diskSpaceCleanForciblyRatio = + Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceCleanForciblyRatio", "0.85")); + + @Override + protected void deleteExpiredFiles() { + + long minOffset = RocksDBMessageStore.this.commitLog.getMinOffset(); + if (minOffset > this.lastPhysicalMinOffset) { + this.lastPhysicalMinOffset = minOffset; + + boolean spaceFull = isSpaceToDelete(); + boolean timeUp = cleanCommitLogService.isTimeToDelete(); + if (spaceFull || timeUp) { + RocksDBMessageStore.this.consumeQueueStore.cleanExpired(minOffset); + } + + RocksDBMessageStore.this.indexService.deleteExpiredFile(minOffset); + } + } + + private boolean isSpaceToDelete() { + double ratio = RocksDBMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0; + + String storePathLogics = StorePathConfigHelper + .getStorePathConsumeQueue(RocksDBMessageStore.this.getMessageStoreConfig().getStorePathRootDir()); + double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics); + if (logicsRatio > diskSpaceWarningLevelRatio) { + boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskFull(); + if (diskOk) { + RocksDBMessageStore.LOGGER.error("logics disk maybe full soon " + logicsRatio + ", so mark disk full"); + } + } else if (logicsRatio > diskSpaceCleanForciblyRatio) { + } else { + boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskOK(); + if (!diskOk) { + RocksDBMessageStore.LOGGER.info("logics disk space OK " + logicsRatio + ", so mark disk ok"); + } + } + + if (logicsRatio < 0 || logicsRatio > ratio) { + RocksDBMessageStore.LOGGER.info("logics disk maybe full soon, so reclaim space, " + logicsRatio); + return true; + } + + return false; + } + } + + class RocksDBFlushConsumeQueueService extends FlushConsumeQueueService { + /** + * There is no need to flush consume queue, + * we put all consume queues in RocksDBConsumeQueueStore, + * it depends on rocksdb to flush consume queue to disk(sorted string table), + * we even don't flush WAL of consume store, since we think it can recover consume queue from commitlog. + */ + @Override + public void run() { + + } + } + + class RocksDBCorrectLogicOffsetService extends CorrectLogicOffsetService { + /** + * There is no need to correct min offset of consume queue, we already fix this problem. + * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset + */ + public void run() { + + } + } + + @Override + public long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter) { + // todo + return 0; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java index 2ae6879aa..91fcb155a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java +++ b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java @@ -30,6 +30,8 @@ public class RunningFlags { private static final int FENCED_BIT = 1 << 5; + private static final int LOGIC_DISK_FULL_BIT = 1 << 5; + private volatile int flagBits = 0; public RunningFlags() { @@ -63,6 +65,10 @@ public class RunningFlags { return result; } + public void clearLogicsQueueError() { + this.flagBits &= ~WRITE_LOGICS_QUEUE_ERROR_BIT; + } + public boolean getAndMakeWriteable() { boolean result = this.isWriteable(); if (!result) { @@ -72,7 +78,7 @@ public class RunningFlags { } public boolean isWriteable() { - if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT)) == 0) { + if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT | LOGIC_DISK_FULL_BIT)) == 0) { return true; } @@ -81,7 +87,7 @@ public class RunningFlags { //for consume queue, just ignore the DISK_FULL_BIT public boolean isCQWriteable() { - if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT)) == 0) { + if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT | LOGIC_DISK_FULL_BIT)) == 0) { return true; } @@ -139,4 +145,16 @@ public class RunningFlags { this.flagBits &= ~DISK_FULL_BIT; return result; } + + public boolean getAndMakeLogicDiskFull() { + boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT); + this.flagBits |= LOGIC_DISK_FULL_BIT; + return result; + } + + public boolean getAndMakeLogicDiskOK() { + boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT); + this.flagBits &= ~LOGIC_DISK_FULL_BIT; + return result; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 9fa448043..028facbdc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -397,8 +397,10 @@ public class MessageStoreConfig { private int batchDispatchRequestThreadPoolNums = 16; // rocksdb mode + private long cleanRocksDBDirtyCQIntervalMin = 60; + private long statRocksDBCQIntervalSec = 10; + private long memTableFlushIntervalMs = 60 * 60 * 1000L; private boolean realTimePersistRocksDBConfig = true; - private long memTableFlushInterval = 60 * 60 * 1000L; private boolean enableRocksDBLog = false; private int topicQueueLockNum = 32; @@ -499,6 +501,10 @@ public class MessageStoreConfig { this.mappedFileSizeCommitLog = mappedFileSizeCommitLog; } + public boolean isEnableRocksDBStore() { + return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.storeType); + } + public String getStoreType() { return storeType; } @@ -508,7 +514,6 @@ public class MessageStoreConfig { } public int getMappedFileSizeConsumeQueue() { - int factor = (int) Math.ceil(this.mappedFileSizeConsumeQueue / (ConsumeQueue.CQ_STORE_UNIT_SIZE * 1.0)); return (int) (factor * ConsumeQueue.CQ_STORE_UNIT_SIZE); } @@ -1738,12 +1743,28 @@ public class MessageStoreConfig { this.realTimePersistRocksDBConfig = realTimePersistRocksDBConfig; } - public long getMemTableFlushInterval() { - return memTableFlushInterval; + public long getStatRocksDBCQIntervalSec() { + return statRocksDBCQIntervalSec; + } + + public void setStatRocksDBCQIntervalSec(long statRocksDBCQIntervalSec) { + this.statRocksDBCQIntervalSec = statRocksDBCQIntervalSec; + } + + public long getCleanRocksDBDirtyCQIntervalMin() { + return cleanRocksDBDirtyCQIntervalMin; + } + + public void setCleanRocksDBDirtyCQIntervalMin(long cleanRocksDBDirtyCQIntervalMin) { + this.cleanRocksDBDirtyCQIntervalMin = cleanRocksDBDirtyCQIntervalMin; + } + + public long getMemTableFlushIntervalMs() { + return memTableFlushIntervalMs; } - public void setMemTableFlushInterval(long memTableFlushInterval) { - this.memTableFlushInterval = memTableFlushInterval; + public void setMemTableFlushIntervalMs(long memTableFlushIntervalMs) { + this.memTableFlushIntervalMs = memTableFlushIntervalMs; } public boolean isEnableRocksDBLog() { diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index d5f6acdc0..70371d83b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -16,26 +16,13 @@ */ package org.apache.rocketmq.store.dledger; -import io.openmessaging.storage.dledger.AppendFuture; -import io.openmessaging.storage.dledger.BatchAppendFuture; -import io.openmessaging.storage.dledger.DLedgerConfig; -import io.openmessaging.storage.dledger.DLedgerServer; -import io.openmessaging.storage.dledger.entry.DLedgerEntry; -import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; -import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; -import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest; -import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; -import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; -import io.openmessaging.storage.dledger.store.file.MmapFile; -import io.openmessaging.storage.dledger.store.file.MmapFileList; -import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; -import io.openmessaging.storage.dledger.utils.DLedgerUtils; import java.net.Inet6Address; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; import java.util.concurrent.CompletableFuture; + import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBatch; @@ -54,6 +41,22 @@ import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.StoreStatsService; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.logfile.MappedFile; +import org.rocksdb.RocksDBException; + +import io.openmessaging.storage.dledger.AppendFuture; +import io.openmessaging.storage.dledger.BatchAppendFuture; +import io.openmessaging.storage.dledger.DLedgerConfig; +import io.openmessaging.storage.dledger.DLedgerServer; +import io.openmessaging.storage.dledger.entry.DLedgerEntry; +import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; +import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; +import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest; +import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; +import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; +import io.openmessaging.storage.dledger.store.file.MmapFile; +import io.openmessaging.storage.dledger.store.file.MmapFileList; +import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; +import io.openmessaging.storage.dledger.utils.DLedgerUtils; /** * Store all metadata downtime for recovery, data protection reliability @@ -269,7 +272,7 @@ public class DLedgerCommitLog extends CommitLog { return null; } - + @Override public boolean getData(final long offset, final int size, final ByteBuffer byteBuffer) { if (offset < dividedCommitlogOffset) { @@ -287,7 +290,7 @@ public class DLedgerCommitLog extends CommitLog { return false; } - private void recover(long maxPhyOffsetOfConsumeQueue) { + private void recover(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { dLedgerFileStore.load(); if (dLedgerFileList.getMappedFiles().size() > 0) { dLedgerFileStore.recover(); @@ -341,12 +344,12 @@ public class DLedgerCommitLog extends CommitLog { } @Override - public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { + public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { recover(maxPhyOffsetOfConsumeQueue); } @Override - public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { + public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { recover(maxPhyOffsetOfConsumeQueue); } @@ -469,9 +472,6 @@ public class DLedgerCommitLog extends CommitLog { String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.getData().length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock); - } catch (Exception e) { - log.error("Put message error", e); - return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); } finally { beginTimeInDledgerLock = 0; putMessageLock.unlock(); @@ -482,6 +482,9 @@ public class DLedgerCommitLog extends CommitLog { } defaultMessageStore.increaseOffset(msg, getMessageNum(msg)); + } catch (Exception e) { + log.error("Put message error", e); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); } finally { topicQueueLock.unlock(topicQueueKey); } @@ -611,9 +614,6 @@ public class DLedgerCommitLog extends CommitLog { appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen, msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock); appendResult.setMsgNum(msgNum); - } catch (Exception e) { - log.error("Put message error", e); - return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); } finally { beginTimeInDledgerLock = 0; putMessageLock.unlock(); @@ -626,7 +626,10 @@ public class DLedgerCommitLog extends CommitLog { defaultMessageStore.increaseOffset(messageExtBatch, (short) batchNum); - } finally { + } catch (Exception e) { + log.error("Put message error", e); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { topicQueueLock.unlock(encodeResult.queueOffsetKey); } diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java index 467da603d..aaea7d690 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; import org.apache.rocketmq.store.CommitLog; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.rocksdb.RocksDBException; public interface HAService { @@ -53,7 +54,7 @@ public interface HAService { * * @param masterEpoch the new masterEpoch */ - default boolean changeToMaster(int masterEpoch) { + default boolean changeToMaster(int masterEpoch) throws RocksDBException { return false; } diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java index 936db0c4c..176c25a96 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java @@ -432,7 +432,7 @@ public class AutoSwitchHAClient extends ServiceThread implements HAClient { /** * Compare the master and slave's epoch file, find consistent point, do truncate. */ - private boolean doTruncate(List masterEpochEntries, long masterEndOffset) throws IOException { + private boolean doTruncate(List masterEpochEntries, long masterEndOffset) throws Exception { if (this.epochCache.getEntrySize() == 0) { // If epochMap is empty, means the broker is a new replicas LOGGER.info("Slave local epochCache is empty, skip truncate log"); diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java index f20bc3e28..64dad9aef 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java @@ -51,6 +51,7 @@ import org.apache.rocketmq.store.ha.GroupTransferService; import org.apache.rocketmq.store.ha.HAClient; import org.apache.rocketmq.store.ha.HAConnection; import org.apache.rocketmq.store.ha.HAConnectionStateNotificationService; +import org.rocksdb.RocksDBException; /** * SwitchAble ha service, support switch role to master or slave. @@ -111,7 +112,7 @@ public class AutoSwitchHAService extends DefaultHAService { } @Override - public boolean changeToMaster(int masterEpoch) { + public boolean changeToMaster(int masterEpoch) throws RocksDBException { final int lastEpoch = this.epochCache.lastEpoch(); if (masterEpoch < lastEpoch) { LOGGER.warn("newMasterEpoch {} < lastEpoch {}, fail to change to master", masterEpoch, lastEpoch); @@ -315,7 +316,7 @@ public class AutoSwitchHAService extends DefaultHAService { final EpochEntry currentLeaderEpoch = this.epochCache.lastEntry(); if (slaveMaxOffset >= currentLeaderEpoch.getStartOffset()) { LOGGER.info("The slave {} has caught up, slaveMaxOffset: {}, confirmOffset: {}, epoch: {}, leader epoch startOffset: {}.", - slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset()); + slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset()); currentSyncStateSet.add(slaveBrokerId); markSynchronizingSyncStateSet(currentSyncStateSet); // Notify the upper layer that syncStateSet changed. @@ -491,7 +492,7 @@ public class AutoSwitchHAService extends DefaultHAService { /** * Try to truncate incomplete msg transferred from master. */ - public long truncateInvalidMsg() { + public long truncateInvalidMsg() throws RocksDBException { long dispatchBehind = this.defaultMessageStore.dispatchBehindBytes(); if (dispatchBehind <= 0) { LOGGER.info("Dispatch complete, skip truncate"); diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java index ab9fc6da7..2f2ce9812 100644 --- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java @@ -17,10 +17,6 @@ package org.apache.rocketmq.store.plugin; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.sdk.metrics.InstrumentSelector; -import io.opentelemetry.sdk.metrics.ViewBuilder; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.LinkedList; @@ -55,10 +51,16 @@ import org.apache.rocketmq.store.hook.PutMessageHook; import org.apache.rocketmq.store.hook.SendMessageBackHook; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; -import org.apache.rocketmq.store.queue.ConsumeQueueStore; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.util.PerfCounter; +import org.rocksdb.RocksDBException; + +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.ViewBuilder; public abstract class AbstractPluginMessageStore implements MessageStore { protected MessageStore next = null; @@ -457,7 +459,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { } @Override - public boolean truncateFiles(long offsetToTruncate) { + public boolean truncateFiles(long offsetToTruncate) throws RocksDBException { return next.truncateFiles(offsetToTruncate); } @@ -511,7 +513,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { @Override public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, - boolean isRecover, boolean isFileEnd) { + boolean isRecover, boolean isFileEnd) throws RocksDBException { next.onCommitLogDispatch(dispatchRequest, doDispatch, commitLogFile, isRecover, isFileEnd); } @@ -551,15 +553,10 @@ public abstract class AbstractPluginMessageStore implements MessageStore { } @Override - public void truncateDirtyLogicFiles(long phyOffset) { + public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException { next.truncateDirtyLogicFiles(phyOffset); } - @Override - public void destroyLogics() { - next.destroyLogics(); - } - @Override public void unlockMappedFile(MappedFile unlockMappedFile) { next.unlockMappedFile(unlockMappedFile); @@ -571,7 +568,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { } @Override - public ConsumeQueueStore getQueueStore() { + public ConsumeQueueStoreInterface getQueueStore() { return next.getQueueStore(); } @@ -586,7 +583,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { } @Override - public void assignOffset(MessageExtBrokerInner msg) { + public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException { next.assignOffset(msg); } @@ -649,4 +646,19 @@ public abstract class AbstractPluginMessageStore implements MessageStore { public void initMetrics(Meter meter, Supplier attributesBuilderSupplier) { next.initMetrics(meter, attributesBuilderSupplier); } + + @Override + public void finishCommitLogDispatch() { + next.finishCommitLogDispatch(); + } + + @Override + public void recoverTopicQueueTable() { + next.recoverTopicQueueTable(); + } + + @Override + public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { + next.notifyMessageArriveIfNecessary(dispatchRequest); + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java new file mode 100644 index 000000000..30054fa50 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.rocksdb.RocksDBException; + +public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface { + protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + + protected final DefaultMessageStore messageStore; + protected final MessageStoreConfig messageStoreConfig; + protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator(); + protected final ConcurrentMap> consumeQueueTable; + + public AbstractConsumeQueueStore(DefaultMessageStore messageStore) { + this.messageStore = messageStore; + this.messageStoreConfig = messageStore.getMessageStoreConfig(); + this.consumeQueueTable = new ConcurrentHashMap<>(32); + } + + @Override + public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) { + consumeQueue.putMessagePositionInfoWrapper(request); + } + + @Override + public Long getMaxOffset(String topic, int queueId) { + return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); + } + + @Override + public void setTopicQueueTable(ConcurrentMap topicQueueTable) { + this.queueOffsetOperator.setTopicQueueTable(topicQueueTable); + this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable); + } + + @Override + public ConcurrentMap getTopicQueueTable() { + return this.queueOffsetOperator.getTopicQueueTable(); + } + + @Override + public void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException { + ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); + consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg); + } + + @Override + public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) { + ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); + consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum); + } + + @Override + public void removeTopicQueueTable(String topic, Integer queueId) { + this.queueOffsetOperator.remove(topic, queueId); + } + + @Override + public ConcurrentMap> getConsumeQueueTable() { + return this.consumeQueueTable; + } + + @Override + public ConcurrentMap findConsumeQueueMap(String topic) { + return this.consumeQueueTable.get(topic); + } + + @Override + public long getStoreTime(CqUnit cqUnit) { + if (cqUnit != null) { + try { + final long phyOffset = cqUnit.getPos(); + final int size = cqUnit.getSize(); + long storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size); + return storeTime; + } catch (Exception e) { + } + } + return -1; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java index 387c233bf..7108c835c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentSkipListMap; import java.util.function.Function; import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.attribute.CQType; import org.apache.rocketmq.common.constant.LoggerName; @@ -311,6 +312,11 @@ public class BatchConsumeQueue implements ConsumeQueueInterface { return new BatchConsumeQueueIterator(sbr); } + @Override + public ReferredIterator iterateFrom(long startIndex, int count) { + return iterateFrom(startIndex); + } + @Override public CqUnit get(long offset) { ReferredIterator it = iterateFrom(offset); @@ -320,6 +326,20 @@ public class BatchConsumeQueue implements ConsumeQueueInterface { return it.nextAndRelease(); } + @Override + public Pair getCqUnitAndStoreTime(long index) { + CqUnit cqUnit = get(index); + Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); + return new Pair<>(cqUnit, messageStoreTime); + } + + @Override + public Pair getEarliestUnitAndStoreTime() { + CqUnit cqUnit = getEarliestUnit(); + Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); + return new Pair<>(cqUnit, messageStoreTime); + } + @Override public CqUnit getEarliestUnit() { return get(minOffsetInQueue); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java index 55d080829..c65f2a68b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java @@ -18,10 +18,12 @@ package org.apache.rocketmq.store.queue; import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.attribute.CQType; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.MessageFilter; +import org.rocksdb.RocksDBException; public interface ConsumeQueueInterface extends FileQueueLifeCycle { /** @@ -44,6 +46,16 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { */ ReferredIterator iterateFrom(long startIndex); + /** + * Get the units from the start offset. + * + * @param startIndex start index + * @param count the unit counts will be iterated + * @return the unit iterateFrom + * @throws RocksDBException only in rocksdb mode + */ + ReferredIterator iterateFrom(long startIndex, int count) throws RocksDBException; + /** * Get cq unit at specified index * @param index index @@ -51,6 +63,18 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { */ CqUnit get(long index); + /** + * Get earliest cq unit + * @return the cq unit and message storeTime at index + */ + Pair getCqUnitAndStoreTime(long index); + + /** + * Get earliest cq unit + * @return earliest cq unit and message storeTime + */ + Pair getEarliestUnitAndStoreTime(); + /** * Get earliest cq unit * @return earliest cq unit @@ -153,8 +177,9 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { * Assign queue offset. * @param queueOffsetAssigner the delegated queue offset assigner * @param msg message itself + * @throws RocksDBException only in rocksdb mode */ - void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg); + void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg) throws RocksDBException; /** diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java index d03d15d65..616511b67 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java @@ -16,64 +16,128 @@ */ package org.apache.rocketmq.store.queue; +import java.io.File; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.attribute.CQType; -import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.QueueTypeUtils; import org.apache.rocketmq.common.utils.ThreadUtils; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.CommitLog; import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; -import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.SelectMappedBufferResult; -import org.apache.rocketmq.store.config.MessageStoreConfig; - -import java.io.File; -import java.util.Iterator; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import static java.lang.String.format; import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue; import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue; -public class ConsumeQueueStore { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); +public class ConsumeQueueStore extends AbstractConsumeQueueStore { - protected final DefaultMessageStore messageStore; - protected final MessageStoreConfig messageStoreConfig; - protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator(); - protected final ConcurrentMap> consumeQueueTable; + public ConsumeQueueStore(DefaultMessageStore messageStore) { + super(messageStore); + } - public ConsumeQueueStore(DefaultMessageStore messageStore, MessageStoreConfig messageStoreConfig) { - this.messageStore = messageStore; - this.messageStoreConfig = messageStoreConfig; - this.consumeQueueTable = new ConcurrentHashMap<>(32); + @Override + public void start() { + log.info("Default ConsumeQueueStore start!"); } - private FileQueueLifeCycle getLifeCycle(String topic, int queueId) { - return findOrCreateConsumeQueue(topic, queueId); + @Override + public boolean load() { + boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ); + boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ); + return cqLoadResult && bcqLoadResult; + } + + @Override + public boolean loadAfterDestroy() { + return true; + } + + @Override + public void recover() { + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueueInterface logic : maps.values()) { + this.recover(logic); + } + } } + @Override + public boolean recoverConcurrently() { + int count = 0; + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + count += maps.values().size(); + } + final CountDownLatch countDownLatch = new CountDownLatch(count); + BlockingQueue recoverQueue = new LinkedBlockingQueue<>(); + final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_"); + List> result = new ArrayList<>(count); + try { + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (final ConsumeQueueInterface logic : maps.values()) { + FutureTask futureTask = new FutureTask<>(() -> { + boolean ret = true; + try { + logic.recover(); + } catch (Throwable e) { + ret = false; + log.error("Exception occurs while recover consume queue concurrently, " + + "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e); + } finally { + countDownLatch.countDown(); + } + return ret; + }); + + result.add(futureTask); + executor.submit(futureTask); + } + } + countDownLatch.await(); + for (FutureTask task : result) { + if (task != null && task.isDone()) { + if (!task.get()) { + return false; + } + } + } + } catch (Exception e) { + log.error("Exception occurs while recover consume queue concurrently", e); + return false; + } finally { + executor.shutdown(); + } + return true; + } + + @Override + public boolean shutdown() { + return true; + } + + @Override public long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.rollNextFile(offset); @@ -83,32 +147,53 @@ public class ConsumeQueueStore { consumeQueue.correctMinOffset(minCommitLogOffset); } - /** - * Apply the dispatched request and build the consume queue. This function should be idempotent. - * - * @param consumeQueue consume queue - * @param request dispatch request - */ - public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) { - consumeQueue.putMessagePositionInfoWrapper(request); - } - + @Override public void putMessagePositionInfoWrapper(DispatchRequest dispatchRequest) { ConsumeQueueInterface cq = this.findOrCreateConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId()); this.putMessagePositionInfoWrapper(cq, dispatchRequest); } + @Override + public List rangeQuery(String topic, int queueId, long startIndex, int num) { + return null; + } + + @Override + public ByteBuffer get(String topic, int queueId, long startIndex) { + return null; + } + + @Override + public long getMaxOffsetInQueue(String topic, int queueId) { + ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getMaxOffsetInQueue(); + } + return 0; + } + + @Override + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) { + ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); + if (logic != null) { + long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType); + // Make sure the result offset is in valid range. + resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue()); + resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue()); + return resultOffset; + } + return 0; + } + + private FileQueueLifeCycle getLifeCycle(String topic, int queueId) { + return findOrCreateConsumeQueue(topic, queueId); + } + public boolean load(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.load(); } - public boolean load() { - boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ); - boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ); - return cqLoadResult && bcqLoadResult; - } - private boolean loadConsumeQueues(String storePath, CQType cqType) { File dirLogic = new File(storePath); File[] fileTopicList = dirLogic.listFiles(); @@ -189,62 +274,17 @@ public class ConsumeQueueStore { fileQueueLifeCycle.recover(); } - public void recover() { - for (ConcurrentMap maps : this.consumeQueueTable.values()) { - for (ConsumeQueueInterface logic : maps.values()) { - this.recover(logic); - } - } - } - - public boolean recoverConcurrently() { - int count = 0; - for (ConcurrentMap maps : this.consumeQueueTable.values()) { - count += maps.values().size(); - } - final CountDownLatch countDownLatch = new CountDownLatch(count); - BlockingQueue recoverQueue = new LinkedBlockingQueue<>(); - final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_"); - List> result = new ArrayList<>(count); - try { - for (ConcurrentMap maps : this.consumeQueueTable.values()) { - for (final ConsumeQueueInterface logic : maps.values()) { - FutureTask futureTask = new FutureTask<>(() -> { - boolean ret = true; - try { - logic.recover(); - } catch (Throwable e) { - ret = false; - log.error("Exception occurs while recover consume queue concurrently, " + - "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e); - } finally { - countDownLatch.countDown(); - } - return ret; - }); - - result.add(futureTask); - executor.submit(futureTask); - } - } - countDownLatch.await(); - for (FutureTask task : result) { - if (task != null && task.isDone()) { - if (!task.get()) { - return false; - } - } - } - } catch (Exception e) { - log.error("Exception occurs while recover consume queue concurrently", e); - return false; - } finally { - executor.shutdown(); + @Override + public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) { + ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getMaxPhysicOffset(); } - return true; + return null; } - public long getMaxOffsetInConsumeQueue() { + @Override + public long getMaxPhyOffsetInConsumeQueue() { long maxPhysicOffset = -1L; for (ConcurrentMap maps : this.consumeQueueTable.values()) { for (ConsumeQueueInterface logic : maps.values()) { @@ -256,11 +296,22 @@ public class ConsumeQueueStore { return maxPhysicOffset; } + @Override + public long getMinOffsetInQueue(String topic, int queueId) { + ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getMinOffsetInQueue(); + } + + return -1; + } + public void checkSelf(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); fileQueueLifeCycle.checkSelf(); } + @Override public void checkSelf() { for (Map.Entry> topicEntry : this.consumeQueueTable.entrySet()) { for (Map.Entry cqEntry : topicEntry.getValue().entrySet()) { @@ -269,16 +320,19 @@ public class ConsumeQueueStore { } } + @Override public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.flush(flushLeastPages); } + @Override public void destroy(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); fileQueueLifeCycle.destroy(); } + @Override public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.deleteExpiredFile(minCommitLogPos); @@ -300,21 +354,20 @@ public class ConsumeQueueStore { fileQueueLifeCycle.cleanSwappedMap(forceCleanSwapIntervalMs); } + @Override public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.isFirstFileAvailable(); } + @Override public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.isFirstFileExist(); } + @Override public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) { - return doFindOrCreateConsumeQueue(topic, queueId); - } - - private ConsumeQueueInterface doFindOrCreateConsumeQueue(String topic, int queueId) { ConcurrentMap map = consumeQueueTable.get(topic); if (null == map) { ConcurrentMap newMap = new ConcurrentHashMap<>(128); @@ -361,46 +414,15 @@ public class ConsumeQueueStore { return logic; } - public Long getMaxOffset(String topic, int queueId) { - return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); - } - - public void setTopicQueueTable(ConcurrentMap topicQueueTable) { - this.queueOffsetOperator.setTopicQueueTable(topicQueueTable); - this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable); - } - - public ConcurrentMap getTopicQueueTable() { - return this.queueOffsetOperator.getTopicQueueTable(); - } - public void setBatchTopicQueueTable(ConcurrentMap batchTopicQueueTable) { this.queueOffsetOperator.setBatchTopicQueueTable(batchTopicQueueTable); } - public void assignQueueOffset(MessageExtBrokerInner msg) { - ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); - consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg); - } - - public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) { - ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); - consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum); - } - public void updateQueueOffset(String topic, int queueId, long offset) { String topicQueueKey = topic + "-" + queueId; this.queueOffsetOperator.updateQueueOffset(topicQueueKey, offset); } - public void removeTopicQueueTable(String topic, Integer queueId) { - this.queueOffsetOperator.remove(topic, queueId); - } - - public ConcurrentMap> getConsumeQueueTable() { - return consumeQueueTable; - } - private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueueInterface consumeQueue) { ConcurrentMap map = this.consumeQueueTable.get(topic); if (null == map) { @@ -412,6 +434,7 @@ public class ConsumeQueueStore { } } + @Override public void recoverOffsetTable(long minPhyOffset) { ConcurrentMap cqOffsetTable = new ConcurrentHashMap<>(1024); ConcurrentMap bcqOffsetTable = new ConcurrentHashMap<>(1024); @@ -431,7 +454,7 @@ public class ConsumeQueueStore { } } - //Correct unSubmit consumeOffset + // Correct unSubmit consumeOffset if (messageStoreConfig.isDuplicationEnable()) { SelectMappedBufferResult lastBuffer = null; long startReadOffset = messageStore.getCommitLog().getConfirmOffset() == -1 ? 0 : messageStore.getCommitLog().getConfirmOffset(); @@ -476,6 +499,7 @@ public class ConsumeQueueStore { this.setBatchTopicQueueTable(bcqOffsetTable); } + @Override public void destroy() { for (ConcurrentMap maps : this.consumeQueueTable.values()) { for (ConsumeQueueInterface logic : maps.values()) { @@ -484,8 +508,9 @@ public class ConsumeQueueStore { } } + @Override public void cleanExpired(long minCommitLogOffset) { - Iterator>> it = this.consumeQueueTable.entrySet().iterator(); + Iterator>> it = this.consumeQueueTable.entrySet().iterator(); while (it.hasNext()) { Map.Entry> next = it.next(); String topic = next.getKey(); @@ -526,14 +551,16 @@ public class ConsumeQueueStore { } } - public void truncateDirty(long phyOffset) { + @Override + public void truncateDirty(long offsetToTruncate) { for (ConcurrentMap maps : this.consumeQueueTable.values()) { for (ConsumeQueueInterface logic : maps.values()) { - this.truncateDirtyLogicFiles(logic, phyOffset); + this.truncateDirtyLogicFiles(logic, offsetToTruncate); } } } + @Override public long getTotalSize() { long totalSize = 0; for (ConcurrentMap maps : this.consumeQueueTable.values()) { diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java new file mode 100644 index 000000000..268803dcc --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.store.DispatchRequest; +import org.rocksdb.RocksDBException; + +public interface ConsumeQueueStoreInterface { + + /** + * Start the consumeQueueStore + */ + void start(); + + /** + * Load from file. + * @return true if loaded successfully. + */ + boolean load(); + + /** + * load after destroy + */ + boolean loadAfterDestroy(); + + /** + * Recover from file. + */ + void recover(); + + /** + * Recover concurrently from file. + * @return true if recovered successfully. + */ + boolean recoverConcurrently(); + + /** + * Shutdown the consumeQueueStore + * @return true if shutdown successfully. + */ + boolean shutdown(); + + /** + * destroy all consumeQueues + */ + void destroy(); + + /** + * destroy the specific consumeQueue + * @throws RocksDBException only in rocksdb mode + */ + void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException; + + /** + * Flush cache to file. + * @param consumeQueue the consumeQueue will be flushed + * @param flushLeastPages the minimum number of pages to be flushed + * @return true if any data has been flushed. + */ + boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages); + + /** + * clean expired data from minPhyOffset + * @param minPhyOffset + */ + void cleanExpired(long minPhyOffset); + + /** + * Check files. + */ + void checkSelf(); + + /** + * Delete expired files ending at min commit log position. + * @param consumeQueue + * @param minCommitLogPos min commit log position + * @return deleted file numbers. + */ + int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos); + + /** + * Is the first file available? + * @param consumeQueue + * @return true if it's available + */ + boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue); + + /** + * Does the first file exist? + * @param consumeQueue + * @return true if it exists + */ + boolean isFirstFileExist(ConsumeQueueInterface consumeQueue); + + /** + * Roll to next file. + * @param consumeQueue + * @param offset next beginning offset + * @return the beginning offset of the next file + */ + long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset); + + /** + * truncate dirty data + * @param offsetToTruncate + * @throws RocksDBException only in rocksdb mode + */ + void truncateDirty(long offsetToTruncate) throws RocksDBException; + + /** + * Apply the dispatched request and build the consume queue. This function should be idempotent. + * + * @param consumeQueue consume queue + * @param request dispatch request + */ + void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request); + + /** + * Apply the dispatched request. This function should be idempotent. + * + * @param request dispatch request + * @throws RocksDBException only in rocksdb mode will throw exception + */ + void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException; + + /** + * range query cqUnit(ByteBuffer) in rocksdb + * @param topic + * @param queueId + * @param startIndex + * @param num + * @return the byteBuffer list of the topic-queueId in rocksdb + * @throws RocksDBException only in rocksdb mode + */ + List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException; + + /** + * query cqUnit(ByteBuffer) in rocksdb + * @param topic + * @param queueId + * @param startIndex + * @return the byteBuffer of the topic-queueId in rocksdb + * @throws RocksDBException only in rocksdb mode + */ + ByteBuffer get(final String topic, final int queueId, final long startIndex) throws RocksDBException; + + /** + * get consumeQueue table + * @return the consumeQueue table + */ + ConcurrentMap> getConsumeQueueTable(); + + /** + * Assign queue offset. + * @param msg message itself + * @throws RocksDBException only in rocksdb mode + */ + void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException; + + /** + * Increase queue offset. + * @param msg message itself + * @param messageNum message number + */ + void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum); + + /** + * recover topicQueue table by minPhyOffset + * @param minPhyOffset + */ + void recoverOffsetTable(long minPhyOffset); + + /** + * set topicQueue table + * @param topicQueueTable + */ + void setTopicQueueTable(ConcurrentMap topicQueueTable); + + /** + * remove topic-queueId from topicQueue table + * @param topic + * @param queueId + */ + void removeTopicQueueTable(String topic, Integer queueId); + + /** + * get topicQueue table + * @return the topicQueue table + */ + ConcurrentMap getTopicQueueTable(); + + /** + * get the max physical offset in consumeQueue + * @param topic + * @param queueId + * @return + */ + Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId); + + /** + * get maxOffset of specific topic-queueId in topicQueue table + * @param topic + * @param queueId + * @return the max offset in QueueOffsetOperator + */ + Long getMaxOffset(String topic, int queueId); + + /** + * get max physic offset in consumeQueue + * @return the max physic offset in consumeQueue + * @throws RocksDBException only in rocksdb mode + */ + long getMaxPhyOffsetInConsumeQueue() throws RocksDBException; + + /** + * get min logic offset of specific topic-queueId in consumeQueue + * @param topic + * @param queueId + * @return the min logic offset of specific topic-queueId in consumeQueue + * @throws RocksDBException only in rocksdb mode + */ + long getMinOffsetInQueue(final String topic, final int queueId) throws RocksDBException; + + /** + * get max logic offset of specific topic-queueId in consumeQueue + * @param topic + * @param queueId + * @return the max logic offset of specific topic-queueId in consumeQueue + * @throws RocksDBException only in rocksdb mode + */ + long getMaxOffsetInQueue(final String topic, final int queueId) throws RocksDBException; + + /** + * Get the message whose timestamp is the smallest, greater than or equal to the given time and when there are more + * than one message satisfy the condition, decide which one to return based on boundaryType. + * @param timestamp timestamp + * @param boundaryType Lower or Upper + * @return the offset(index) + * @throws RocksDBException only in rocksdb mode + */ + long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException; + + /** + * find or create the consumeQueue + * @param topic + * @param queueId + * @return the consumeQueue + */ + ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId); + + /** + * find the consumeQueueMap of topic + * @param topic + * @return the consumeQueueMap of topic + */ + ConcurrentMap findConsumeQueueMap(String topic); + + /** + * get the total size of all consumeQueue + * @return the total size of all consumeQueue + */ + long getTotalSize(); + + /** + * Get store time from commitlog by cqUnit + * @param cqUnit + * @return + */ + long getStoreTime(CqUnit cqUnit); +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java new file mode 100644 index 000000000..d6291d908 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.config.MessageStoreConfig; + +public class MultiDispatch { + + public static String lmqQueueKey(String queueName) { + StringBuilder keyBuilder = new StringBuilder(); + keyBuilder.append(queueName); + keyBuilder.append('-'); + int queueId = 0; + keyBuilder.append(queueId); + return keyBuilder.toString(); + } + + public static boolean isNeedHandleMultiDispatch(MessageStoreConfig messageStoreConfig, String topic) { + return messageStoreConfig.isEnableMultiDispatch() + && !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) + && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX) + && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); + } + + public static boolean checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, DispatchRequest dispatchRequest) { + if (!isNeedHandleMultiDispatch(messageStoreConfig, dispatchRequest.getTopic())) { + return false; + } + Map prop = dispatchRequest.getPropertiesMap(); + if (prop == null || prop.isEmpty()) { + return false; + } + String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); + if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { + return false; + } + return true; + } + + public static List checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, List dispatchRequests) { + if (!messageStoreConfig.isEnableMultiDispatch() || dispatchRequests == null || dispatchRequests.size() == 0) { + return null; + } + List result = new ArrayList<>(); + for (DispatchRequest dispatchRequest : dispatchRequests) { + if (checkMultiDispatchQueue(messageStoreConfig, dispatchRequest)) { + result.add(dispatchRequest); + } + } + return dispatchRequests; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java index 2545bbf52..8da374828 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java @@ -41,6 +41,10 @@ public class QueueOffsetOperator { return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); } + public Long getTopicQueueNextOffset(String topicQueueKey) { + return this.topicQueueTable.get(topicQueueKey); + } + public void increaseQueueOffset(String topicQueueKey, short messageNum) { Long queueOffset = ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); topicQueueTable.put(topicQueueKey, queueOffset + messageNum); @@ -63,6 +67,10 @@ public class QueueOffsetOperator { return ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); } + public Long getLmqTopicQueueNextOffset(String topicQueueKey) { + return this.lmqTopicQueueTable.get(topicQueueKey); + } + public void increaseLmqOffset(String topicQueueKey, short messageNum) { Long lmqOffset = ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); this.lmqTopicQueueTable.put(topicQueueKey, lmqOffset + messageNum); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java new file mode 100644 index 000000000..759be395d --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.attribute.CQType; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.ConsumeQueue; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.MessageFilter; +import org.apache.rocketmq.store.MessageStore; +import org.rocksdb.RocksDBException; + +public class RocksDBConsumeQueue implements ConsumeQueueInterface { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + + private final MessageStore messageStore; + private final String topic; + private final int queueId; + + public RocksDBConsumeQueue(final MessageStore messageStore, final String topic, final int queueId) { + this.messageStore = messageStore; + this.topic = topic; + this.queueId = queueId; + } + + public RocksDBConsumeQueue(final String topic, final int queueId) { + this.messageStore = null; + this.topic = topic; + this.queueId = queueId; + } + + @Override + public boolean load() { + return true; + } + + @Override + public void recover() { + // ignore + } + + @Override + public void checkSelf() { + // ignore + } + + @Override + public boolean flush(final int flushLeastPages) { + return true; + } + + @Override + public void destroy() { + // ignore + } + + @Override + public void truncateDirtyLogicFiles(long maxCommitLogPos) { + // ignored + } + + @Override + public int deleteExpiredFile(long minCommitLogPos) { + return 0; + } + + @Override + public long rollNextFile(long nextBeginOffset) { + return 0; + } + + @Override + public boolean isFirstFileAvailable() { + return true; + } + + @Override + public boolean isFirstFileExist() { + return true; + } + + @Override + public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) { + // ignore + } + + @Override + public void cleanSwappedMap(long forceCleanSwapIntervalMs) { + // ignore + } + + @Override + public long getMaxOffsetInQueue() { + try { + return this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); + } catch (RocksDBException e) { + ERROR_LOG.error("getMaxOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e); + return 0; + } + } + + @Override + public long getMessageTotalInQueue() { + try { + long maxOffsetInQueue = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); + long minOffsetInQueue = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId); + return maxOffsetInQueue - minOffsetInQueue; + } catch (RocksDBException e) { + ERROR_LOG.error("getMessageTotalInQueue Failed. topic: {}, queueId: {}, {}", topic, queueId, e); + } + return -1; + } + + /** + * We already implement it in RocksDBConsumeQueueStore. + * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime + * @param timestamp timestamp + * @return + */ + @Override + public long getOffsetInQueueByTime(long timestamp) { + return 0; + } + + /** + * We already implement it in RocksDBConsumeQueueStore. + * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime + * @param timestamp timestamp + * @param boundaryType Lower or Upper + * @return + */ + @Override + public long getOffsetInQueueByTime(long timestamp, BoundaryType boundaryType) { + return 0; + } + + @Override + public long getMaxPhysicOffset() { + Long maxPhyOffset = this.messageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue(topic, queueId); + return maxPhyOffset == null ? -1 : maxPhyOffset; + } + + @Override + public long getMinLogicOffset() { + return 0; + } + + @Override + public CQType getCQType() { + return CQType.RocksDBCQ; + } + + @Override + public long getTotalSize() { + // ignored + return 0; + } + + @Override + public int getUnitSize() { + // attention: unitSize should equal to 'ConsumeQueue.CQ_STORE_UNIT_SIZE' + return ConsumeQueue.CQ_STORE_UNIT_SIZE; + } + + /** + * Ignored, we already implement this method + * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) + */ + @Override + public void correctMinOffset(long minCommitLogOffset) { + + } + + /** + * Ignored, in rocksdb mode, we build cq in RocksDBConsumeQueueStore + * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore#putMessagePosition() + */ + @Override + public void putMessagePositionInfoWrapper(DispatchRequest request) { + + } + + @Override + public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg) throws RocksDBException { + String topicQueueKey = getTopic() + "-" + getQueueId(); + Long queueOffset = queueOffsetOperator.getTopicQueueNextOffset(topicQueueKey); + if (queueOffset == null) { + // we will recover topic queue table from rocksdb when we use it. + queueOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); + queueOffsetOperator.updateQueueOffset(topicQueueKey, queueOffset); + } + msg.setQueueOffset(queueOffset); + + // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), + // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. + if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { + return; + } + String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + if (StringUtils.isBlank(multiDispatchQueue)) { + return; + } + String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + Long[] queueOffsets = new Long[queues.length]; + for (int i = 0; i < queues.length; i++) { + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { + String key = MultiDispatch.lmqQueueKey(queues[i]); + queueOffsets[i] = queueOffsetOperator.getLmqTopicQueueNextOffset(key); + } + } + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, + StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); + msg.removeWaitStorePropertyString(); + } + + @Override + public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg, short messageNum) { + String topicQueueKey = getTopic() + "-" + getQueueId(); + queueOffsetOperator.increaseQueueOffset(topicQueueKey, messageNum); + + // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), + // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. + if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { + return; + } + String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + if (StringUtils.isBlank(multiDispatchQueue)) { + return; + } + String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + for (int i = 0; i < queues.length; i++) { + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { + String key = MultiDispatch.lmqQueueKey(queues[i]); + queueOffsetOperator.increaseLmqOffset(key, (short) 1); + } + } + } + + @Override + public long estimateMessageCount(long from, long to, MessageFilter filter) { + // todo + return 0; + } + + @Override + public long getMinOffsetInQueue() { + return this.messageStore.getMinOffsetInQueue(this.topic, this.queueId); + } + + private int pullNum(long cqOffset, long maxCqOffset) { + long diffLong = maxCqOffset - cqOffset; + if (diffLong < Integer.MAX_VALUE) { + int diffInt = (int) diffLong; + return diffInt > 16 ? 16 : diffInt; + } + return 16; + } + + @Override + public ReferredIterator iterateFrom(final long startIndex) { + try { + long maxCqOffset = getMaxOffsetInQueue(); + if (startIndex < maxCqOffset) { + int num = pullNum(startIndex, maxCqOffset); + return iterateFrom0(startIndex, num); + } + } catch (RocksDBException e) { + log.error("[RocksDBConsumeQueue] iterateFrom error!", e); + } + return null; + } + + @Override + public ReferredIterator iterateFrom(long startIndex, int count) throws RocksDBException { + long maxCqOffset = getMaxOffsetInQueue(); + if (startIndex < maxCqOffset) { + int num = Math.min((int)(maxCqOffset - startIndex), count); + return iterateFrom0(startIndex, num); + } + return null; + } + + @Override + public CqUnit get(long index) { + Pair pair = getCqUnitAndStoreTime(index); + return pair == null ? null : pair.getObject1(); + } + + @Override + public Pair getCqUnitAndStoreTime(long index) { + ByteBuffer byteBuffer; + try { + byteBuffer = this.messageStore.getQueueStore().get(topic, queueId, index); + } catch (RocksDBException e) { + ERROR_LOG.error("getUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e); + return null; + } + if (byteBuffer == null || byteBuffer.remaining() < RocksDBConsumeQueueTable.CQ_UNIT_SIZE) { + return null; + } + long phyOffset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + long tagCode = byteBuffer.getLong(); + long messageStoreTime = byteBuffer.getLong(); + return new Pair<>(new CqUnit(index, phyOffset, size, tagCode), messageStoreTime); + } + + @Override + public Pair getEarliestUnitAndStoreTime() { + try { + long minOffset = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId); + return getCqUnitAndStoreTime(minOffset); + } catch (RocksDBException e) { + ERROR_LOG.error("getEarliestUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e); + } + return null; + } + + @Override + public CqUnit getEarliestUnit() { + Pair pair = getEarliestUnitAndStoreTime(); + return pair == null ? null : pair.getObject1(); + } + + @Override + public CqUnit getLatestUnit() { + try { + long maxOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); + return get(maxOffset); + } catch (RocksDBException e) { + ERROR_LOG.error("getLatestUnit Failed. topic: {}, queueId: {}, {}", topic, queueId, e.getMessage()); + } + return null; + } + + @Override + public long getLastOffset() { + return getMaxPhysicOffset(); + } + + private ReferredIterator iterateFrom0(final long startIndex, final int count) throws RocksDBException { + List byteBufferList = this.messageStore.getQueueStore().rangeQuery(topic, queueId, startIndex, count); + if (byteBufferList == null || byteBufferList.isEmpty()) { + if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + log.warn("iterateFrom0 - find nothing, startIndex:{}, count:{}", startIndex, count); + } + return null; + } + return new RocksDBConsumeQueueIterator(byteBufferList, startIndex); + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public int getQueueId() { + return queueId; + } + + private class RocksDBConsumeQueueIterator implements ReferredIterator { + private final List byteBufferList; + private final long startIndex; + private final int totalCount; + private int currentIndex; + + public RocksDBConsumeQueueIterator(final List byteBufferList, final long startIndex) { + this.byteBufferList = byteBufferList; + this.startIndex = startIndex; + this.totalCount = byteBufferList.size(); + this.currentIndex = 0; + } + + @Override + public boolean hasNext() { + return this.currentIndex < this.totalCount; + } + + @Override + public CqUnit next() { + if (!hasNext()) { + return null; + } + final int currentIndex = this.currentIndex; + final ByteBuffer byteBuffer = this.byteBufferList.get(currentIndex); + CqUnit cqUnit = new CqUnit(this.startIndex + currentIndex, byteBuffer.getLong(), byteBuffer.getInt(), byteBuffer.getLong()); + this.currentIndex++; + return cqUnit; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove"); + } + + @Override + public void release() { + } + + @Override + public CqUnit nextAndRelease() { + try { + return next(); + } finally { + release(); + } + } + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java new file mode 100644 index 000000000..6fa66282e --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java @@ -0,0 +1,641 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.ConsumeQueue; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; + +import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; + +public class RocksDBConsumeQueueOffsetTable { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + + private static final byte[] MAX_BYTES = "max".getBytes(CHARSET_UTF8); + private static final byte[] MIN_BYTES = "min".getBytes(CHARSET_UTF8); + + /** + * Rocksdb ConsumeQueue's Offset unit. Format: + * + * 

+     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬─────────────┐
+     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  Max(Min) │  CTRL_1   │   QueueId   │
+     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │  (4 Bytes)  │
+     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴─────────────┤
+     * │                                                    Key Unit                                                   │
+     * │                                                                                                               │
+     *
+ * + * 
+     * ┌─────────────────────────────┬────────────────────────┐
+     * │  CommitLog Physical Offset  │   ConsumeQueue Offset  │
+     * │        (8 Bytes)            │    (8 Bytes)           │
+     * ├─────────────────────────────┴────────────────────────┤
+     * │                     Value Unit                       │
+     * │                                                      │
+     *
+ * ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) = 16 Bytes + */ + private static final int OFFSET_PHY_OFFSET = 0; + private static final int OFFSET_CQ_OFFSET = 8; + /** + * + * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐ + * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ Max(Min) │ CTRL_1 │ QueueId │ + * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │ + * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴─────────────┤ + */ + private static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4; + private static final int OFFSET_VALUE_LENGTH = 8 + 8; + + /** + * We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread. + * @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread. + * If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of + * RocksDBConsumeQueueOffsetTable to find it. + */ + private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC; + private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(CHARSET_UTF8); + private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length; + private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN); + private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN]; + private final ByteBuffer maxPhyOffsetBB; + static { + buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true); + INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); + INNER_CHECKPOINT_TOPIC.get(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY); + } + + private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; + private final ConsumeQueueRocksDBStorage rocksDBStorage; + private final DefaultMessageStore messageStore; + + private ColumnFamilyHandle offsetCFH; + + /** + * Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them + * from heap to avoid accessing rocksdb. + * @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset --> topicQueueMaxCqOffset + * @see ConsumeQueue#getMinLogicOffset(), minLogicOffset --> topicQueueMinOffset + */ + private final Map topicQueueMinOffset; + private final Map topicQueueMaxCqOffset; + + public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable, + ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { + this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable; + this.rocksDBStorage = rocksDBStorage; + this.messageStore = messageStore; + this.topicQueueMinOffset = new ConcurrentHashMap(1024); + this.topicQueueMaxCqOffset = new ConcurrentHashMap(1024); + + this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8); + } + + public void load() { + this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle(); + } + + public void updateTempTopicQueueMaxOffset(final Pair offsetBBPair, + final byte[] topicBytes, final DispatchRequest request, + final Map> tempTopicQueueMaxOffsetMap) { + buildOffsetKeyAndValueByteBuffer(offsetBBPair, topicBytes, request); + ByteBuffer topicQueueId = offsetBBPair.getObject1(); + ByteBuffer maxOffsetBB = offsetBBPair.getObject2(); + Pair old = tempTopicQueueMaxOffsetMap.get(topicQueueId); + if (old == null) { + tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair(maxOffsetBB, request)); + } else { + long oldMaxOffset = old.getObject1().getLong(OFFSET_CQ_OFFSET); + long maxOffset = maxOffsetBB.getLong(OFFSET_CQ_OFFSET); + if (maxOffset >= oldMaxOffset) { + ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset); + } + } + } + + public void putMaxPhyAndCqOffset(final Map> tempTopicQueueMaxOffsetMap, + final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { + for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { + writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1()); + } + + appendMaxPhyOffset(writeBatch, maxPhyOffset); + } + + public void putHeapMaxCqOffset(final Map> tempTopicQueueMaxOffsetMap) { + for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { + DispatchRequest request = entry.getValue().getObject2(); + putHeapMaxCqOffset(request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset()); + } + } + + /** + * When topic is deleted, we clean up its offset info in rocksdb. + * @param topic + * @param queueId + * @throws RocksDBException + */ + public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false); + byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array()); + Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; + + final ByteBuffer maxOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, true); + byte[] maxOffsetBytes = this.rocksDBStorage.getOffset(maxOffsetKey.array()); + Long endCQOffset = (maxOffsetBytes != null) ? ByteBuffer.wrap(maxOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; + + writeBatch.delete(this.offsetCFH, minOffsetKey.array()); + writeBatch.delete(this.offsetCFH, maxOffsetKey.array()); + + String topicQueueId = buildTopicQueueId(topic, queueId); + removeHeapMinCqOffset(topicQueueId); + removeHeapMaxCqOffset(topicQueueId); + + log.info("RocksDB offset table delete topic: {}, queueId: {}, minOffset: {}, maxOffset: {}", topic, queueId, + startCQOffset, endCQOffset); + } + + private void appendMaxPhyOffset(final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { + final ByteBuffer maxPhyOffsetBB = this.maxPhyOffsetBB; + maxPhyOffsetBB.position(0).limit(8); + maxPhyOffsetBB.putLong(maxPhyOffset); + maxPhyOffsetBB.flip(); + + INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); + writeBatch.put(this.offsetCFH, INNER_CHECKPOINT_TOPIC, maxPhyOffsetBB); + } + + public long getMaxPhyOffset() throws RocksDBException { + byte[] valueBytes = this.rocksDBStorage.getOffset(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY); + if (valueBytes == null) { + return 0; + } + ByteBuffer valueBB = ByteBuffer.wrap(valueBytes); + return valueBB.getLong(0); + } + + /** + * Traverse the offset table to find dirty topic + * @param existTopicSet + * @return + */ + public Map> iterateOffsetTable2FindDirty(final Set existTopicSet) { + Map> topicQueueIdToBeDeletedMap = new HashMap<>(); + + RocksIterator iterator = null; + try { + iterator = rocksDBStorage.seekOffsetCF(); + if (iterator == null) { + return topicQueueIdToBeDeletedMap; + } + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + byte[] key = iterator.key(); + byte[] value = iterator.value(); + if (key == null || key.length <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + || value == null || value.length != OFFSET_VALUE_LENGTH) { + continue; + } + ByteBuffer keyBB = ByteBuffer.wrap(key); + int topicLen = keyBB.getInt(0); + byte[] topicBytes = new byte[topicLen]; + /** + * "Topic Bytes Array Size" + "CTRL_1" = 4 + 1 + */ + keyBB.position(4 + 1); + keyBB.get(topicBytes); + String topic = new String(topicBytes, CHARSET_UTF8); + if (TopicValidator.isSystemTopic(topic)) { + continue; + } + + /** + * "Topic Bytes Array Size" + "CTRL_1" + "Topic Bytes Array" + "CTRL_1" + "Max(min)" + "CTRL_1" + * = 4 + 1 + topicLen + 1 + 3 + 1 + */ + int queueId = keyBB.getInt(4 + 1 + topicLen + 1 + 3 + 1); + + if (!existTopicSet.contains(topic)) { + ByteBuffer valueBB = ByteBuffer.wrap(value); + long cqOffset = valueBB.getLong(OFFSET_CQ_OFFSET); + + Set topicQueueIdSet = topicQueueIdToBeDeletedMap.get(topic); + if (topicQueueIdSet == null) { + Set newSet = new HashSet<>(); + newSet.add(queueId); + topicQueueIdToBeDeletedMap.put(topic, newSet); + } else { + topicQueueIdSet.add(queueId); + } + ERROR_LOG.info("RocksDBConsumeQueueOffsetTable has dirty cqOffset. topic: {}, queueId: {}, cqOffset: {}", + topic, queueId, cqOffset); + } + } + } catch (Exception e) { + ERROR_LOG.error("iterateOffsetTable2MarkDirtyCQ Failed.", e); + } finally { + if (iterator != null) { + iterator.close(); + } + } + return topicQueueIdToBeDeletedMap; + } + + public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException { + Long maxCqOffset = getHeapMaxCqOffset(topic, queueId); + + if (maxCqOffset == null) { + final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); + maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null; + String topicQueueId = buildTopicQueueId(topic, queueId); + this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, maxCqOffset != null ? maxCqOffset : -1L); + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, maxCqOffset); + } + } + + return maxCqOffset; + } + + /** + * truncate dirty offset in rocksdb + * @param offsetToTruncate + * @throws RocksDBException + */ + public void truncateDirty(long offsetToTruncate) throws RocksDBException { + correctMaxPyhOffset(offsetToTruncate); + + ConcurrentMap allTopicConfigMap = this.messageStore.getTopicConfigs(); + if (allTopicConfigMap == null) { + return; + } + for (TopicConfig topicConfig : allTopicConfigMap.values()) { + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + truncateDirtyOffset(topicConfig.getTopicName(), i); + } + } + } + + private Pair isMinOffsetOk(final String topic, final int queueId, final long minPhyOffset) throws RocksDBException { + PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); + if (phyAndCQOffset != null) { + final long phyOffset = phyAndCQOffset.getPhyOffset(); + final long cqOffset = phyAndCQOffset.getCqOffset(); + + return (phyOffset >= minPhyOffset) ? new Pair(true, cqOffset) : new Pair(false, cqOffset); + } + ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId); + if (byteBuffer == null) { + return new Pair(false, 0L); + } + final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); + final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); + if (phyOffset >= minPhyOffset) { + String topicQueueId = buildTopicQueueId(topic, queueId); + PhyAndCQOffset newPhyAndCQOffset = new PhyAndCQOffset(phyOffset, cqOffset); + this.topicQueueMinOffset.putIfAbsent(topicQueueId, newPhyAndCQOffset); + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("updateMinOffsetInQueue. {}, {}", topicQueueId, newPhyAndCQOffset); + } + return new Pair(true, cqOffset); + } + return new Pair(false, cqOffset); + } + + private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException { + final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); + if (byteBuffer == null) { + return; + } + + long maxPhyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); + long maxCqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); + long maxPhyOffsetInCQ = getMaxPhyOffset(); + + if (maxPhyOffset >= maxPhyOffsetInCQ) { + correctMaxCqOffset(topic, queueId, maxCqOffset, maxPhyOffsetInCQ); + Long newMaxCqOffset = getHeapMaxCqOffset(topic, queueId); + ROCKSDB_LOG.warn("truncateDirtyLogicFile topic: {}, queueId: {} from {} to {}", topic, queueId, + maxPhyOffset, newMaxCqOffset); + } + } + + private void correctMaxPyhOffset(long maxPhyOffset) throws RocksDBException { + if (!this.rocksDBStorage.hold()) { + return; + } + try { + WriteBatch writeBatch = new WriteBatch(); + long oldMaxPhyOffset = getMaxPhyOffset(); + if (oldMaxPhyOffset <= maxPhyOffset) { + return; + } + log.info("correctMaxPyhOffset, oldMaxPhyOffset={}, newMaxPhyOffset={}", oldMaxPhyOffset, maxPhyOffset); + appendMaxPhyOffset(writeBatch, maxPhyOffset); + this.rocksDBStorage.batchPut(writeBatch); + } catch (RocksDBException e) { + ERROR_LOG.error("correctMaxPyhOffset Failed.", e); + throw e; + } finally { + this.rocksDBStorage.release(); + } + } + + public long getMinCqOffset(String topic, int queueId) throws RocksDBException { + final long minPhyOffset = this.messageStore.getMinPhyOffset(); + Pair pair = isMinOffsetOk(topic, queueId, minPhyOffset); + final long cqOffset = pair.getObject2(); + if (!pair.getObject1() && correctMinCqOffset(topic, queueId, cqOffset, minPhyOffset)) { + PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); + if (phyAndCQOffset != null) { + if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("getMinOffsetInQueue miss heap. topic: {}, queueId: {}, old: {}, new: {}", + topic, queueId, cqOffset, phyAndCQOffset); + } + return phyAndCQOffset.getCqOffset(); + } + } + return cqOffset; + } + + public Long getMaxPhyOffset(String topic, int queueId) { + try { + ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); + if (byteBuffer != null) { + return byteBuffer.getLong(OFFSET_PHY_OFFSET); + } + } catch (Exception e) { + ERROR_LOG.info("getMaxPhyOffset error. topic: {}, queueId: {}", topic, queueId); + } + return null; + } + + private ByteBuffer getMinPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException { + return getPhyAndCqOffsetInKV(topic, queueId, false); + } + + private ByteBuffer getMaxPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException { + return getPhyAndCqOffsetInKV(topic, queueId, true); + } + + private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max); + + byte[] value = this.rocksDBStorage.getOffset(keyBB.array()); + return (value != null) ? ByteBuffer.wrap(value) : null; + } + + private String buildTopicQueueId(final String topic, final int queueId) { + return topic + "-" + queueId; + } + + private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, final long minCQOffset) { + String topicQueueId = buildTopicQueueId(topic, queueId); + PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset); + this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset); + } + + private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxCQOffset) { + String topicQueueId = buildTopicQueueId(topic, queueId); + Long oldMaxCqOffset = this.topicQueueMaxCqOffset.put(topicQueueId, maxCQOffset); + if (oldMaxCqOffset != null && oldMaxCqOffset > maxCQOffset) { + ERROR_LOG.error("cqOffset invalid0. old: {}, now: {}", oldMaxCqOffset, maxCQOffset); + } + } + + private PhyAndCQOffset getHeapMinOffset(final String topic, final int queueId) { + return this.topicQueueMinOffset.get(buildTopicQueueId(topic, queueId)); + } + + private Long getHeapMaxCqOffset(final String topic, final int queueId) { + String topicQueueId = buildTopicQueueId(topic, queueId); + return this.topicQueueMaxCqOffset.get(topicQueueId); + } + + private PhyAndCQOffset removeHeapMinCqOffset(String topicQueueId) { + return this.topicQueueMinOffset.remove(topicQueueId); + } + + private Long removeHeapMaxCqOffset(String topicQueueId) { + return this.topicQueueMaxCqOffset.remove(topicQueueId); + } + + private void updateCqOffset(final String topic, final int queueId, final long phyOffset, + final long cqOffset, boolean max) throws RocksDBException { + if (!this.rocksDBStorage.hold()) { + return; + } + WriteBatch writeBatch = new WriteBatch(); + try { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max); + + final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset); + writeBatch.put(this.offsetCFH, offsetKey.array(), offsetValue.array()); + this.rocksDBStorage.batchPut(writeBatch); + + if (max) { + putHeapMaxCqOffset(topic, queueId, cqOffset); + } else { + putHeapMinCqOffset(topic, queueId, phyOffset, cqOffset); + } + } catch (RocksDBException e) { + ERROR_LOG.error("updateCqOffset({}) failed.", max ? "max" : "min", e); + throw e; + } finally { + writeBatch.close(); + this.rocksDBStorage.release(); + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}", + max ? "max" : "min", topic, queueId, phyOffset, cqOffset); + } + } + } + + private boolean correctMaxCqOffset(final String topic, final int queueId, final long maxCQOffset, + final long maxPhyOffsetInCQ) throws RocksDBException { + // 'getMinOffsetInQueue' may correct minCqOffset and put it into heap + long minCQOffset = getMinCqOffset(topic, queueId); + PhyAndCQOffset minPhyAndCQOffset = getHeapMinOffset(topic, queueId); + if (minPhyAndCQOffset == null + || minPhyAndCQOffset.getCqOffset() != minCQOffset + || minPhyAndCQOffset.getPhyOffset() > maxPhyOffsetInCQ) { + ROCKSDB_LOG.info("[BUG] correctMaxCqOffset error! topic: {}, queueId: {}, maxPhyOffsetInCQ: {}, " + + "minCqOffset: {}, phyAndCQOffset: {}", + topic, queueId, maxPhyOffsetInCQ, minCQOffset, minPhyAndCQOffset); + throw new RocksDBException("correctMaxCqOffset error"); + } + + long high = maxCQOffset; + long low = minCQOffset; + PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, + low, maxPhyOffsetInCQ, false); + + long targetCQOffset = targetPhyAndCQOffset.getCqOffset(); + long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset(); + + if (targetCQOffset == -1) { + if (maxCQOffset != minCQOffset) { + updateCqOffset(topic, queueId, minPhyAndCQOffset.getPhyOffset(), minCQOffset, true); + } + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyAndCQOffset.getPhyOffset()); + } + return false; + } else { + updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, true); + return true; + } + } + + private boolean correctMinCqOffset(final String topic, final int queueId, + final long minCQOffset, final long minPhyOffset) throws RocksDBException { + final ByteBuffer maxBB = getMaxPhyAndCqOffsetInKV(topic, queueId); + if (maxBB == null) { + updateCqOffset(topic, queueId, minPhyOffset, 0L, false); + return true; + } + final long maxPhyOffset = maxBB.getLong(OFFSET_PHY_OFFSET); + final long maxCQOffset = maxBB.getLong(OFFSET_CQ_OFFSET); + + if (maxPhyOffset < minPhyOffset) { + updateCqOffset(topic, queueId, minPhyOffset, maxCQOffset + 1, false); + return true; + } + + long high = maxCQOffset; + long low = minCQOffset; + PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, low, + minPhyOffset, true); + long targetCQOffset = phyAndCQOffset.getCqOffset(); + long targetPhyOffset = phyAndCQOffset.getPhyOffset(); + + if (targetCQOffset == -1) { + if (maxCQOffset != minCQOffset) { + updateCqOffset(topic, queueId, maxPhyOffset, maxCQOffset, false); + } + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyOffset); + } + return false; + } else { + updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, false); + return true; + } + } + + public static Pair getOffsetByteBufferPair() { + ByteBuffer offsetKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN); + ByteBuffer offsetValue = ByteBuffer.allocateDirect(OFFSET_VALUE_LENGTH); + return new Pair<>(offsetKey, offsetValue); + } + + private void buildOffsetKeyAndValueByteBuffer(final Pair offsetBBPair, + final byte[] topicBytes, final DispatchRequest request) { + final ByteBuffer offsetKey = offsetBBPair.getObject1(); + buildOffsetKeyByteBuffer(offsetKey, topicBytes, request.getQueueId(), true); + + final ByteBuffer offsetValue = offsetBBPair.getObject2(); + buildOffsetValueByteBuffer(offsetValue, request.getCommitLogOffset(), request.getConsumeQueueOffset()); + } + + private ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) { + ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); + return byteBuffer; + } + + private void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) { + byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); + } + + private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, + final boolean max) { + byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1); + if (max) { + byteBuffer.put(MAX_BYTES); + } else { + byteBuffer.put(MIN_BYTES); + } + byteBuffer.put(CTRL_1).putInt(queueId); + byteBuffer.flip(); + } + + private void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { + byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH); + buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); + } + + private ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH); + buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); + return byteBuffer; + } + + private void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { + byteBuffer.putLong(phyOffset).putLong(cqOffset); + byteBuffer.flip(); + } + + static class PhyAndCQOffset { + private final long phyOffset; + private final long cqOffset; + + public PhyAndCQOffset(final long phyOffset, final long cqOffset) { + this.phyOffset = phyOffset; + this.cqOffset = cqOffset; + } + + public long getPhyOffset() { + return this.phyOffset; + } + + public long getCqOffset() { + return this.cqOffset; + } + + @Override + public String toString() { + return "[cqOffset=" + cqOffset + ", phyOffset=" + phyOffset + "]"; + } + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java new file mode 100644 index 000000000..78456cfcd --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; + +public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { + private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + + public static final byte CTRL_0 = '\u0000'; + public static final byte CTRL_1 = '\u0001'; + public static final byte CTRL_2 = '\u0002'; + + private static final int BATCH_SIZE = 16; + public static final int MAX_KEY_LEN = 300; + + private final ScheduledExecutorService scheduledExecutorService; + private final String storePath; + + /** + * we use two tables with different ColumnFamilyHandle, called RocksDBConsumeQueueTable and RocksDBConsumeQueueOffsetTable. + * 1.RocksDBConsumeQueueTable uses to store CqUnit[physicalOffset, msgSize, tagHashCode, msgStoreTime] + * 2.RocksDBConsumeQueueOffsetTable uses to store physicalOffset and consumeQueueOffset(@see PhyAndCQOffset) of topic-queueId + */ + private final ConsumeQueueRocksDBStorage rocksDBStorage; + private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; + private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable; + + private final WriteBatch writeBatch; + private final List bufferDRList; + private final List> cqBBPairList; + private final List> offsetBBPairList; + private final Map> tempTopicQueueMaxOffsetMap; + private volatile boolean isCQError = false; + + public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { + super(messageStore); + + this.storePath = StorePathConfigHelper.getStorePathConsumeQueue(messageStoreConfig.getStorePathRootDir()); + this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath, 4); + this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore); + this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); + + this.writeBatch = new WriteBatch(); + this.bufferDRList = new ArrayList(BATCH_SIZE); + this.cqBBPairList = new ArrayList(BATCH_SIZE); + this.offsetBBPairList = new ArrayList(BATCH_SIZE); + for (int i = 0; i < BATCH_SIZE; i++) { + this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); + this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); + } + + this.tempTopicQueueMaxOffsetMap = new HashMap<>(); + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("RocksDBConsumeQueueStoreScheduledThread", messageStore.getBrokerIdentity())); + } + + @Override + public void start() { + log.info("RocksDB ConsumeQueueStore start!"); + this.scheduledExecutorService.scheduleAtFixedRate(() -> { + this.rocksDBStorage.statRocksdb(ROCKSDB_LOG); + }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS); + + this.scheduledExecutorService.scheduleWithFixedDelay(() -> { + cleanDirty(messageStore.getTopicConfigs().keySet()); + }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES); + } + + private void cleanDirty(final Set existTopicSet) { + try { + Map> topicQueueIdToBeDeletedMap = + this.rocksDBConsumeQueueOffsetTable.iterateOffsetTable2FindDirty(existTopicSet); + + for (Map.Entry> entry : topicQueueIdToBeDeletedMap.entrySet()) { + String topic = entry.getKey(); + for (int queueId : entry.getValue()) { + destroy(new RocksDBConsumeQueue(topic, queueId)); + } + } + } catch (Exception e) { + log.error("cleanUnusedTopic Failed.", e); + } + } + + @Override + public boolean load() { + boolean result = this.rocksDBStorage.start(); + this.rocksDBConsumeQueueTable.load(); + this.rocksDBConsumeQueueOffsetTable.load(); + log.info("load rocksdb consume queue {}.", result ? "OK" : "Failed"); + return result; + } + + @Override + public boolean loadAfterDestroy() { + return this.load(); + } + + @Override + public void recover() { + // ignored + } + + @Override + public boolean recoverConcurrently() { + return true; + } + + @Override + public boolean shutdown() { + this.scheduledExecutorService.shutdown(); + return shutdownInner(); + } + + private boolean shutdownInner() { + return this.rocksDBStorage.shutdown(); + } + + @Override + public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException { + if (request == null || this.bufferDRList.size() >= BATCH_SIZE) { + putMessagePosition(); + } + if (request != null) { + this.bufferDRList.add(request); + } + } + + public void putMessagePosition() throws RocksDBException { + final int maxRetries = 30; + for (int i = 0; i < maxRetries; i++) { + if (putMessagePosition0()) { + if (this.isCQError) { + this.messageStore.getRunningFlags().clearLogicsQueueError(); + this.isCQError = false; + } + return; + } else { + ERROR_LOG.warn("{} put cq Failed. retryTime: {}", i); + try { + Thread.sleep(100); + } catch (InterruptedException ignored) { + } + } + } + if (!this.isCQError) { + ERROR_LOG.error("[BUG] put CQ Failed."); + this.messageStore.getRunningFlags().makeLogicsQueueError(); + this.isCQError = true; + } + throw new RocksDBException("put CQ Failed"); + } + + private boolean putMessagePosition0() { + if (!this.rocksDBStorage.hold()) { + return false; + } + + final Map> tempTopicQueueMaxOffsetMap = this.tempTopicQueueMaxOffsetMap; + try { + final List bufferDRList = this.bufferDRList; + final int size = bufferDRList.size(); + if (size == 0) { + return true; + } + final List> cqBBPairList = this.cqBBPairList; + final List> offsetBBPairList = this.offsetBBPairList; + final WriteBatch writeBatch = this.writeBatch; + + long maxPhyOffset = 0; + for (int i = size - 1; i >= 0; i--) { + final DispatchRequest request = bufferDRList.get(i); + final byte[] topicBytes = request.getTopic().getBytes(DataConverter.CHARSET_UTF8); + + this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(cqBBPairList.get(i), topicBytes, request, writeBatch); + this.rocksDBConsumeQueueOffsetTable.updateTempTopicQueueMaxOffset(offsetBBPairList.get(i), + topicBytes, request, tempTopicQueueMaxOffsetMap); + + final int msgSize = request.getMsgSize(); + final long phyOffset = request.getCommitLogOffset(); + if (phyOffset + msgSize >= maxPhyOffset) { + maxPhyOffset = phyOffset + msgSize; + } + } + + this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset); + + // clear writeBatch in batchPut + this.rocksDBStorage.batchPut(writeBatch); + + this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap); + + long storeTimeStamp = bufferDRList.get(size - 1).getStoreTimestamp(); + if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE + || this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) { + this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp); + } + this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp); + + notifyMessageArriveAndClear(); + return true; + } catch (Exception e) { + ERROR_LOG.error("putMessagePosition0 Failed.", e); + return false; + } finally { + tempTopicQueueMaxOffsetMap.clear(); + this.rocksDBStorage.release(); + } + } + + private void notifyMessageArriveAndClear() { + final List bufferDRList = this.bufferDRList; + try { + for (DispatchRequest dp : bufferDRList) { + this.messageStore.notifyMessageArriveIfNecessary(dp); + } + } catch (Exception e) { + ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e); + } finally { + bufferDRList.clear(); + } + } + + @Override + public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { + return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num); + } + + @Override + public ByteBuffer get(final String topic, final int queueId, final long cqOffset) throws RocksDBException { + return this.rocksDBConsumeQueueTable.getCQInKV(topic, queueId, cqOffset); + } + + /** + * Ignored, we do not need to recover topicQueueTable and correct minLogicOffset. Because we will correct them + * when we use them, we call it lazy correction. + * @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short) + * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) + */ + @Override + public void recoverOffsetTable(long minPhyOffset) { + + } + + @Override + public void destroy() { + try { + shutdownInner(); + FileUtils.deleteDirectory(new File(this.storePath)); + } catch (Exception e) { + ERROR_LOG.error("destroy cq Failed. {}", this.storePath, e); + } + } + + @Override + public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException { + String topic = consumeQueue.getTopic(); + int queueId = consumeQueue.getQueueId(); + if (StringUtils.isEmpty(topic) || queueId < 0 || !this.rocksDBStorage.hold()) { + return; + } + + WriteBatch writeBatch = new WriteBatch(); + try { + this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch); + this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch); + + this.rocksDBStorage.batchPut(writeBatch); + } catch (RocksDBException e) { + ERROR_LOG.error("kv deleteTopic {} Failed.", topic, e); + throw e; + } finally { + writeBatch.close(); + this.rocksDBStorage.release(); + } + } + + @Override + public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { + try { + this.rocksDBStorage.flushWAL(); + } catch (Exception e) { + } + return true; + } + + @Override + public void checkSelf() { + // ignored + } + + @Override + public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) { + // ignored + return 0; + } + + /** + * We do not need to truncate dirty CQ in RocksDBConsumeQueueTable, Because dirty CQ in RocksDBConsumeQueueTable + * will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted. + * But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in + * RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset). + * @param offsetToTruncate + * @throws RocksDBException + */ + @Override + public void truncateDirty(long offsetToTruncate) throws RocksDBException { + long maxPhyOffsetInRocksdb = getMaxPhyOffsetInConsumeQueue(); + if (offsetToTruncate >= maxPhyOffsetInRocksdb) { + return; + } + + this.rocksDBConsumeQueueOffsetTable.truncateDirty(offsetToTruncate); + } + + @Override + public void cleanExpired(final long minPhyOffset) { + this.rocksDBStorage.manualCompaction(minPhyOffset); + } + + @Override + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException { + final long minPhysicOffset = this.messageStore.getMinPhyOffset(); + long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); + Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); + if (high == null || high == -1) { + return 0; + } + return this.rocksDBConsumeQueueTable.binarySearchInCQByTime(topic, queueId, high, low, timestamp, minPhysicOffset); + } + + @Override + public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException { + Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); + return (maxOffset != null) ? maxOffset + 1 : 0; + } + + @Override + public long getMinOffsetInQueue(String topic, int queueId) throws RocksDBException { + return this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); + } + + @Override + public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) { + return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(topic, queueId); + } + + @Override + public long getMaxPhyOffsetInConsumeQueue() throws RocksDBException { + return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(); + } + + @Override + public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) { + ConcurrentMap map = this.consumeQueueTable.get(topic); + if (null == map) { + ConcurrentMap newMap = new ConcurrentHashMap<>(128); + ConcurrentMap oldMap = this.consumeQueueTable.putIfAbsent(topic, newMap); + if (oldMap != null) { + map = oldMap; + } else { + map = newMap; + } + } + + ConsumeQueueInterface logic = map.get(queueId); + if (logic != null) { + return logic; + } + + ConsumeQueueInterface newLogic = new RocksDBConsumeQueue(this.messageStore, topic, queueId); + ConsumeQueueInterface oldLogic = map.putIfAbsent(queueId, newLogic); + + return oldLogic != null ? oldLogic : newLogic; + } + + @Override + public long rollNextFile(ConsumeQueueInterface consumeQueue, long offset) { + return 0; + } + + @Override + public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) { + return true; + } + + @Override + public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) { + return true; + } + + @Override + public long getTotalSize() { + return 0; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java new file mode 100644 index 000000000..0a735ea27 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset; +import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; + +import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_0; +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_2; + +/** + * We use RocksDBConsumeQueueTable to store cqUnit. + */ +public class RocksDBConsumeQueueTable { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + + /** + * Rocksdb ConsumeQueue's store unit. Format: + * + * 
+     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬───────────────────────┐
+     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  QueueId  │  CTRL_1   │  ConsumeQueue Offset  │
+     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │     (8 Bytes)         │
+     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴───────────────────────┤
+     * │                                                    Key Unit                                                             │
+     * │                                                                                                                         │
+     *
+ * + * 
+     * ┌─────────────────────────────┬───────────────────┬──────────────────┬──────────────────┐
+     * │  CommitLog Physical Offset  │      Body Size    │   Tag HashCode   │  Msg Store Time  │
+     * │        (8 Bytes)            │      (4 Bytes)    │    (8 Bytes)     │    (8 Bytes)     │
+     * ├─────────────────────────────┴───────────────────┴──────────────────┴──────────────────┤
+     * │                                                    Value Unit                         │
+     * │                                                                                       │
+     *
+ * ConsumeQueue's store unit. Size: + * CommitLog Physical Offset(8) + Body Size(4) + Tag HashCode(8) + Msg Store Time(8) = 28 Bytes + */ + private static final int PHY_OFFSET_OFFSET = 0; + private static final int PHY_MSG_LEN_OFFSET = 8; + private static final int MSG_TAG_HASHCODE_OFFSET = 12; + private static final int MSG_STORE_TIME_SIZE_OFFSET = 20; + public static final int CQ_UNIT_SIZE = 8 + 4 + 8 + 8; + + /** + * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬───────────────────────┐ + * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_1 │ ConsumeQueue Offset │ + * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ (8 Bytes) │ + * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴───────────────────────┤ + */ + private static final int CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1 + 8; + + /** + * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────────────┐ + * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_0(CTRL_2) │ + * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ + * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────────────┤ + */ + private static final int DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1; + + private final ConsumeQueueRocksDBStorage rocksDBStorage; + private final DefaultMessageStore messageStore; + + private ColumnFamilyHandle defaultCFH; + + public RocksDBConsumeQueueTable(ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { + this.rocksDBStorage = rocksDBStorage; + this.messageStore = messageStore; + } + + public void load() { + this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle(); + } + + public void buildAndPutCQByteBuffer(final Pair cqBBPair, + final byte[] topicBytes, final DispatchRequest request, final WriteBatch writeBatch) throws RocksDBException { + final ByteBuffer cqKey = cqBBPair.getObject1(); + buildCQKeyByteBuffer(cqKey, topicBytes, request.getQueueId(), request.getConsumeQueueOffset()); + + final ByteBuffer cqValue = cqBBPair.getObject2(); + buildCQValueByteBuffer(cqValue, request.getCommitLogOffset(), request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp()); + + writeBatch.put(this.defaultCFH, cqKey, cqValue); + } + + public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset); + byte[] value = this.rocksDBStorage.getCQ(keyBB.array()); + return (value != null) ? ByteBuffer.wrap(value) : null; + } + + public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final List defaultCFHList = new ArrayList(num); + final ByteBuffer[] resultList = new ByteBuffer[num]; + final List kvIndexList = new ArrayList(num); + final List kvKeyList = new ArrayList(num); + for (int i = 0; i < num; i++) { + final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i); + kvIndexList.add(i); + kvKeyList.add(keyBB.array()); + defaultCFHList.add(this.defaultCFH); + } + int keyNum = kvIndexList.size(); + if (keyNum > 0) { + List kvValueList = this.rocksDBStorage.multiGet(defaultCFHList, kvKeyList); + final int valueNum = kvValueList.size(); + if (keyNum != valueNum) { + throw new RocksDBException("rocksdb bug, multiGet"); + } + for (int i = 0; i < valueNum; i++) { + byte[] value = kvValueList.get(i); + if (value == null) { + continue; + } + ByteBuffer byteBuffer = ByteBuffer.wrap(value); + resultList[kvIndexList.get(i)] = byteBuffer; + } + } + + final int resultSize = resultList.length; + List bbValueList = new ArrayList(resultSize); + for (int i = 0; i < resultSize; i++) { + ByteBuffer byteBuffer = resultList[i]; + if (byteBuffer == null) { + break; + } + bbValueList.add(byteBuffer); + } + return bbValueList; + } + + /** + * When topic is deleted, we clean up its CqUnit in rocksdb. + * @param topic + * @param queueId + * @throws RocksDBException + */ + public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId); + final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId); + + writeBatch.deleteRange(this.defaultCFH, cqStartKey.array(), cqEndKey.array()); + + log.info("Rocksdb consumeQueue table delete topic. {}, {}", topic, queueId); + } + + public long binarySearchInCQByTime(String topic, int queueId, long high, long low, long timestamp, + long minPhysicOffset) throws RocksDBException { + long result = 0; + long targetOffset = -1L, leftOffset = -1L, rightOffset = -1L; + long leftValue = -1L, rightValue = -1L; + while (high >= low) { + long midOffset = low + ((high - low) >>> 1); + ByteBuffer byteBuffer = getCQInKV(topic, queueId, midOffset); + if (byteBuffer == null) { + ERROR_LOG.warn("binarySearchInCQByTimeStamp Failed. topic: {}, queueId: {}, timestamp: {}, result: null", + topic, queueId, timestamp); + low = midOffset + 1; + continue; + } + + long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET); + if (phyOffset < minPhysicOffset) { + low = midOffset + 1; + leftOffset = midOffset; + continue; + } + long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET); + if (storeTime < 0) { + return 0; + } else if (storeTime == timestamp) { + targetOffset = midOffset; + break; + } else if (storeTime > timestamp) { + high = midOffset - 1; + rightOffset = midOffset; + rightValue = storeTime; + } else { + low = midOffset + 1; + leftOffset = midOffset; + leftValue = storeTime; + } + } + if (targetOffset != -1) { + result = targetOffset; + } else { + if (leftValue == -1) { + result = rightOffset; + } else if (rightValue == -1) { + result = leftOffset; + } else { + result = Math.abs(timestamp - leftValue) > Math.abs(timestamp - rightValue) ? rightOffset : leftOffset; + } + } + return result; + } + + public PhyAndCQOffset binarySearchInCQ(String topic, int queueId, long high, long low, long targetPhyOffset, + boolean min) throws RocksDBException { + long resultCQOffset = -1L; + long resultPhyOffset = -1L; + while (high >= low) { + long midCQOffset = low + ((high - low) >>> 1); + ByteBuffer byteBuffer = getCQInKV(topic, queueId, midCQOffset); + if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("binarySearchInCQ. {}, {}, {}, {}, {}", topic, queueId, midCQOffset, low, high); + } + if (byteBuffer == null) { + low = midCQOffset + 1; + continue; + } + + final long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET); + if (phyOffset == targetPhyOffset) { + if (min) { + resultCQOffset = midCQOffset; + resultPhyOffset = phyOffset; + } + break; + } else if (phyOffset > targetPhyOffset) { + high = midCQOffset - 1; + if (min) { + resultCQOffset = midCQOffset; + resultPhyOffset = phyOffset; + } + } else { + low = midCQOffset + 1; + if (!min) { + resultCQOffset = midCQOffset; + resultPhyOffset = phyOffset; + } + } + } + return new PhyAndCQOffset(resultPhyOffset, resultCQOffset); + } + + public static Pair getCQByteBufferPair() { + ByteBuffer cqKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN); + ByteBuffer cqValue = ByteBuffer.allocateDirect(CQ_UNIT_SIZE); + return new Pair<>(cqKey, cqValue); + } + + private ByteBuffer buildCQKeyByteBuffer(final byte[] topicBytes, final int queueId, final long cqOffset) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset); + return byteBuffer; + } + + private void buildCQKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) { + byteBuffer.position(0).limit(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset); + } + + private void buildCQKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) { + byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(CTRL_1).putLong(cqOffset); + byteBuffer.flip(); + } + + private void buildCQValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, final long tagsCode, final long storeTimestamp) { + byteBuffer.position(0).limit(CQ_UNIT_SIZE); + buildCQValueByteBuffer0(byteBuffer, phyOffset, msgSize, tagsCode, storeTimestamp); + } + + private void buildCQValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, + final long tagsCode, final long storeTimestamp) { + byteBuffer.putLong(phyOffset).putInt(msgSize).putLong(tagsCode).putLong(storeTimestamp); + byteBuffer.flip(); + } + + private ByteBuffer buildDeleteCQKey(final boolean start, final byte[] topicBytes, final int queueId) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + + byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(start ? CTRL_0 : CTRL_2); + byteBuffer.flip(); + return byteBuffer; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java new file mode 100644 index 000000000..aa796c4d3 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.rocksdb; + +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.MessageStore; +import org.rocksdb.AbstractCompactionFilter; +import org.rocksdb.AbstractCompactionFilterFactory; +import org.rocksdb.RemoveConsumeQueueCompactionFilter; + +public class ConsumeQueueCompactionFilterFactory extends AbstractCompactionFilterFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + private final MessageStore messageStore; + + public ConsumeQueueCompactionFilterFactory(final MessageStore messageStore) { + this.messageStore = messageStore; + } + + @Override + public String name() { + return "ConsumeQueueCompactionFilterFactory"; + } + + @Override + public RemoveConsumeQueueCompactionFilter createCompactionFilter(final AbstractCompactionFilter.Context context) { + long minPhyOffset = this.messageStore.getMinPhyOffset(); + LOGGER.info("manualCompaction minPhyOffset: {}, isFull: {}, isManual: {}", + minPhyOffset, context.isFullCompaction(), context.isManualCompaction()); + return new RemoveConsumeQueueCompactionFilter(minPhyOffset); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java new file mode 100644 index 000000000..362684560 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.rocksdb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.config.AbstractRocksDBStorage; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.store.MessageStore; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.CompactRangeOptions; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; + +public class ConsumeQueueRocksDBStorage extends AbstractRocksDBStorage { + private final MessageStore messageStore; + private volatile ColumnFamilyHandle offsetCFHandle; + + public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath, final int prefixLen) { + this.messageStore = messageStore; + this.dbPath = dbPath; + this.readOnly = false; + } + + private void initOptions() { + this.options = RocksDBOptionsFactory.createDBOptions(); + + this.writeOptions = new WriteOptions(); + this.writeOptions.setSync(false); + this.writeOptions.setDisableWAL(true); + this.writeOptions.setNoSlowdown(true); + + this.totalOrderReadOptions = new ReadOptions(); + this.totalOrderReadOptions.setPrefixSameAsStart(false); + this.totalOrderReadOptions.setTotalOrderSeek(false); + + this.compactRangeOptions = new CompactRangeOptions(); + this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce); + this.compactRangeOptions.setAllowWriteStall(true); + this.compactRangeOptions.setExclusiveManualCompaction(false); + this.compactRangeOptions.setChangeLevel(true); + this.compactRangeOptions.setTargetLevel(-1); + this.compactRangeOptions.setMaxSubcompactions(4); + } + + @Override + protected boolean postLoad() { + try { + UtilAll.ensureDirOK(this.dbPath); + + initOptions(); + + final List cfDescriptors = new ArrayList(); + + ColumnFamilyOptions cqCfOptions = RocksDBOptionsFactory.createCQCFOptions(this.messageStore); + this.cfOptions.add(cqCfOptions); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cqCfOptions)); + + ColumnFamilyOptions offsetCfOptions = RocksDBOptionsFactory.createOffsetCFOptions(); + this.cfOptions.add(offsetCfOptions); + cfDescriptors.add(new ColumnFamilyDescriptor("offset".getBytes(DataConverter.CHARSET_UTF8), offsetCfOptions)); + + final List cfHandles = new ArrayList(); + open(cfDescriptors, cfHandles); + + this.defaultCFHandle = cfHandles.get(0); + this.offsetCFHandle = cfHandles.get(1); + } catch (final Exception e) { + LOGGER.error("postLoad Failed. {}", this.dbPath, e); + return false; + } + return true; + } + + @Override + protected void preShutdown() { + this.offsetCFHandle.close(); + } + + public byte[] getCQ(final byte[] keyBytes) throws RocksDBException { + return get(this.defaultCFHandle, this.totalOrderReadOptions, keyBytes); + } + + public byte[] getOffset(final byte[] keyBytes) throws RocksDBException { + return get(this.offsetCFHandle, this.totalOrderReadOptions, keyBytes); + } + + public List multiGet(final List cfhList, final List keys) throws RocksDBException { + return multiGet(this.totalOrderReadOptions, cfhList, keys); + } + + public void batchPut(final WriteBatch batch) throws RocksDBException { + batchPut(this.writeOptions, batch); + } + + public void manualCompaction(final long minPhyOffset) { + try { + manualCompaction(minPhyOffset, this.compactRangeOptions); + } catch (Exception e) { + LOGGER.error("manualCompaction Failed. minPhyOffset: {}", minPhyOffset, e); + } + } + + public RocksIterator seekOffsetCF() { + return this.db.newIterator(this.offsetCFHandle, this.totalOrderReadOptions); + } + + public ColumnFamilyHandle getOffsetCFHandle() { + return this.offsetCFHandle; + } +} \ No newline at end of file diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java new file mode 100644 index 000000000..a3a99d334 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.rocksdb; + +import org.apache.rocketmq.common.config.ConfigRocksDBStorage; +import org.apache.rocketmq.store.MessageStore; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.CompactionOptionsUniversal; +import org.rocksdb.CompactionStopStyle; +import org.rocksdb.CompactionStyle; +import org.rocksdb.CompressionType; +import org.rocksdb.DBOptions; +import org.rocksdb.DataBlockIndexType; +import org.rocksdb.IndexType; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.LRUCache; +import org.rocksdb.RateLimiter; +import org.rocksdb.SkipListMemTableConfig; +import org.rocksdb.Statistics; +import org.rocksdb.StatsLevel; +import org.rocksdb.StringAppendOperator; +import org.rocksdb.WALRecoveryMode; +import org.rocksdb.util.SizeUnit; + +public class RocksDBOptionsFactory { + + public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageStore) { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). + setFormatVersion(5). + setIndexType(IndexType.kBinarySearch). + setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash). + setDataBlockHashTableUtilRatio(0.75). + setBlockSize(32 * SizeUnit.KB). + setMetadataBlockSize(4 * SizeUnit.KB). + setFilterPolicy(new BloomFilter(16, false)). + setCacheIndexAndFilterBlocks(false). + setCacheIndexAndFilterBlocksWithHighPriority(true). + setPinL0FilterAndIndexBlocksInCache(false). + setPinTopLevelIndexAndFilter(true). + setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false)). + setWholeKeyFiltering(true); + + ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); + CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal(); + compactionOption.setSizeRatio(100). + setMaxSizeAmplificationPercent(25). + setAllowTrivialMove(true). + setMinMergeWidth(2). + setMaxMergeWidth(Integer.MAX_VALUE). + setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize). + setCompressionSizePercent(-1); + return columnFamilyOptions.setMaxWriteBufferNumber(4). + setWriteBufferSize(128 * SizeUnit.MB). + setMinWriteBufferNumberToMerge(1). + setTableFormatConfig(blockBasedTableConfig). + setMemTableConfig(new SkipListMemTableConfig()). + setCompressionType(CompressionType.LZ4_COMPRESSION). + setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION). + setNumLevels(7). + setCompactionStyle(CompactionStyle.UNIVERSAL). + setCompactionOptionsUniversal(compactionOption). + setMaxCompactionBytes(100 * SizeUnit.GB). + setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB). + setHardPendingCompactionBytesLimit(256 * SizeUnit.GB). + setLevel0FileNumCompactionTrigger(2). + setLevel0SlowdownWritesTrigger(8). + setLevel0StopWritesTrigger(10). + setTargetFileSizeBase(256 * SizeUnit.MB). + setTargetFileSizeMultiplier(2). + setMergeOperator(new StringAppendOperator()). + setCompactionFilterFactory(new ConsumeQueueCompactionFilterFactory(messageStore)). + setReportBgIoStats(true). + setOptimizeFiltersForHits(true); + } + + public static ColumnFamilyOptions createOffsetCFOptions() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). + setFormatVersion(5). + setIndexType(IndexType.kBinarySearch). + setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch). + setBlockSize(32 * SizeUnit.KB). + setFilterPolicy(new BloomFilter(16, false)). + setCacheIndexAndFilterBlocks(false). + setCacheIndexAndFilterBlocksWithHighPriority(true). + setPinL0FilterAndIndexBlocksInCache(false). + setPinTopLevelIndexAndFilter(true). + setBlockCache(new LRUCache(128 * SizeUnit.MB, 8, false)). + setWholeKeyFiltering(true); + + ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); + return columnFamilyOptions.setMaxWriteBufferNumber(4). + setWriteBufferSize(64 * SizeUnit.MB). + setMinWriteBufferNumberToMerge(1). + setTableFormatConfig(blockBasedTableConfig). + setMemTableConfig(new SkipListMemTableConfig()). + setCompressionType(CompressionType.NO_COMPRESSION). + setNumLevels(7). + setCompactionStyle(CompactionStyle.LEVEL). + setLevel0FileNumCompactionTrigger(2). + setLevel0SlowdownWritesTrigger(8). + setLevel0StopWritesTrigger(10). + setTargetFileSizeBase(64 * SizeUnit.MB). + setTargetFileSizeMultiplier(2). + setMaxBytesForLevelBase(256 * SizeUnit.MB). + setMaxBytesForLevelMultiplier(2). + setMergeOperator(new StringAppendOperator()). + setInplaceUpdateSupport(true); + } + + /** + * Create a rocksdb db options, the user must take care to close it after closing db. + * @return + */ + public static DBOptions createDBOptions() { + //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide + // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java + DBOptions options = new DBOptions(); + Statistics statistics = new Statistics(); + statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS); + return options. + setDbLogDir(ConfigRocksDBStorage.getDBLogDir()). + setInfoLogLevel(InfoLogLevel.INFO_LEVEL). + setWalRecoveryMode(WALRecoveryMode.PointInTimeRecovery). + setManualWalFlush(true). + setMaxTotalWalSize(0). + setWalSizeLimitMB(0). + setWalTtlSeconds(0). + setCreateIfMissing(true). + setCreateMissingColumnFamilies(true). + setMaxOpenFiles(-1). + setMaxLogFileSize(1 * SizeUnit.GB). + setKeepLogFileNum(5). + setMaxManifestFileSize(1 * SizeUnit.GB). + setAllowConcurrentMemtableWrite(false). + setStatistics(statistics). + setAtomicFlush(true). + setMaxBackgroundJobs(32). + setMaxSubcompactions(8). + setParanoidChecks(true). + setDelayedWriteRate(16 * SizeUnit.MB). + setRateLimiter(new RateLimiter(100 * SizeUnit.MB)). + setUseDirectIoForFlushAndCompaction(false). + setUseDirectReads(false); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index ac4c61cd6..3ab51a26d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -57,7 +57,6 @@ import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; -import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; @@ -66,6 +65,9 @@ import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; +import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +import org.apache.rocketmq.store.queue.CqUnit; +import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.util.PerfCounter; @@ -333,7 +335,7 @@ public class TimerMessageStore { // if not, use cq offset. long msgQueueOffset = messageExt.getQueueOffset(); int queueId = messageExt.getQueueId(); - ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); + ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); if (null == cq) { return msgQueueOffset; } @@ -346,15 +348,18 @@ public class TimerMessageStore { offsetPy, sizePy); break; } - SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(tmpOffset); - if (null == bufferCQ) { - // offset in msg may be greater than offset of cq. - tmpOffset -= 1; - continue; - } + ReferredIterator iterator = null; try { - long offsetPyTemp = bufferCQ.getByteBuffer().getLong(); - int sizePyTemp = bufferCQ.getByteBuffer().getInt(); + iterator = cq.iterateFrom(tmpOffset); + CqUnit cqUnit = null; + if (null == iterator || (cqUnit = iterator.next()) == null) { + // offset in msg may be greater than offset of cq. + tmpOffset -= 1; + continue; + } + + long offsetPyTemp = cqUnit.getPos(); + int sizePyTemp = cqUnit.getSize(); if (offsetPyTemp == offsetPy && sizePyTemp == sizePy) { LOGGER.info("reviseQueueOffset check cq offset ok. {}, {}, {}", tmpOffset, offsetPyTemp, sizePyTemp); @@ -365,7 +370,9 @@ public class TimerMessageStore { } catch (Throwable e) { LOGGER.error("reviseQueueOffset check cq offset error.", e); } finally { - bufferCQ.release(); + if (iterator != null) { + iterator.release(); + } } } @@ -633,7 +640,7 @@ public class TimerMessageStore { if (!isRunningEnqueue()) { return false; } - ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); + ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); if (null == cq) { return false; } @@ -643,18 +650,22 @@ public class TimerMessageStore { currQueueOffset = cq.getMinOffsetInQueue(); } long offset = currQueueOffset; - SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(offset); - if (null == bufferCQ) { - return false; - } + ReferredIterator iterator = null; try { + iterator = cq.iterateFrom(offset); + if (null == iterator) { + return false; + } + int i = 0; - for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) { + while (iterator.hasNext()) { + i++; perfCounterTicks.startTick("enqueue_get"); try { - long offsetPy = bufferCQ.getByteBuffer().getLong(); - int sizePy = bufferCQ.getByteBuffer().getInt(); - bufferCQ.getByteBuffer().getLong(); //tags code + CqUnit cqUnit = iterator.next(); + long offsetPy = cqUnit.getPos(); + int sizePy = cqUnit.getSize(); + cqUnit.getTagsCode(); //tags code MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy); if (null == msgExt) { perfCounterTicks.getCounter("enqueue_get_miss"); @@ -663,7 +674,7 @@ public class TimerMessageStore { lastEnqueueButExpiredStoreTime = msgExt.getStoreTimestamp(); long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); // use CQ offset, not offset in Message - msgExt.setQueueOffset(offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE)); + msgExt.setQueueOffset(offset + i); TimerRequest timerRequest = new TimerRequest(offsetPy, sizePy, delayedTime, System.currentTimeMillis(), MAGIC_DEFAULT, msgExt); // System.out.printf("build enqueue request, %s%n", timerRequest); while (!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) { @@ -687,14 +698,16 @@ public class TimerMessageStore { if (!isRunningEnqueue()) { return false; } - currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); + currQueueOffset = offset + i; } - currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); + currQueueOffset = offset + i; return i > 0; } catch (Exception e) { LOGGER.error("Unknown exception in enqueuing", e); } finally { - bufferCQ.release(); + if (iterator != null) { + iterator.release(); + } } return false; } @@ -1642,7 +1655,7 @@ public class TimerMessageStore { if (System.currentTimeMillis() - start > storeConfig.getTimerProgressLogIntervalMs()) { start = System.currentTimeMillis(); long tmpQueueOffset = currQueueOffset; - ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0); + ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); TimerMessageStore.LOGGER.info("[{}]Timer progress-check commitRead:[{}] currRead:[{}] currWrite:[{}] readBehind:{} currReadOffset:{} offsetBehind:{} behindMaster:{} " + "enqPutQueue:{} deqGetQueue:{} deqPutQueue:{} allCongestNum:{} enqExpiredStoreTime:{}", @@ -1685,7 +1698,7 @@ public class TimerMessageStore { public long getEnqueueBehindMessages() { long tmpQueueOffset = currQueueOffset; - ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0); + ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); return maxOffsetInQueue - tmpQueueOffset; } diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java index 12d1e5723..1d09ca86e 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java @@ -428,9 +428,10 @@ public class DefaultMessageStoreTest { private long getStoreTime(CqUnit cqUnit) { try { - Method getStoreTime = getDefaultMessageStore().getClass().getDeclaredMethod("getStoreTime", CqUnit.class); + Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass(); + Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class); getStoreTime.setAccessible(true); - return (long) getStoreTime.invoke(getDefaultMessageStore(), cqUnit); + return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { throw new RuntimeException(e); } diff --git a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java index 85626a332..2447bbf68 100644 --- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java @@ -28,9 +28,11 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.queue.MultiDispatch; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.rocksdb.RocksDBException; import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue; import static org.junit.Assert.assertEquals; @@ -69,15 +71,15 @@ public class MultiDispatchTest { } @Test - public void queueKey() { + public void lmqQueueKey() { MessageExtBrokerInner messageExtBrokerInner = mock(MessageExtBrokerInner.class); when(messageExtBrokerInner.getQueueId()).thenReturn(2); - String ret = consumeQueue.queueKey("%LMQ%lmq123", messageExtBrokerInner); + String ret = MultiDispatch.lmqQueueKey("%LMQ%lmq123"); assertEquals(ret, "%LMQ%lmq123-0"); } @Test - public void wrapMultiDispatch() { + public void wrapMultiDispatch() throws RocksDBException { MessageExtBrokerInner messageExtBrokerInner = buildMessageMultiQueue(); messageStore.assignOffset(messageExtBrokerInner); assertEquals(messageExtBrokerInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), "0,0"); diff --git a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java new file mode 100644 index 000000000..acf5edf51 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java @@ -0,0 +1,1060 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store; + +import java.io.File; +import java.io.RandomAccessFile; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.common.collect.Sets; + +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.MessageBatch; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageExtBatch; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.config.FlushDiskType; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +import org.apache.rocketmq.store.queue.CqUnit; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.assertj.core.util.Strings; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class RocksDBMessageStoreTest { + private final String storeMessage = "Once, there was a chance for me!"; + private final String messageTopic = "FooBar"; + private final String storeType = StoreType.DEFAULT_ROCKSDB.getStoreType(); + private int queueTotal = 100; + private AtomicInteger queueId = new AtomicInteger(0); + private SocketAddress bornHost; + private SocketAddress storeHost; + private byte[] messageBody; + private MessageStore messageStore; + + @Before + public void init() throws Exception { + if (notExecuted()) { + return; + } + storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + + messageStore = buildMessageStore(); + boolean load = messageStore.load(); + assertTrue(load); + messageStore.start(); + } + + @Test(expected = OverlappingFileLockException.class) + public void test_repeat_restart() throws Exception { + if (notExecuted()) { + throw new OverlappingFileLockException(); + } + queueTotal = 1; + messageBody = storeMessage.getBytes(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8); + messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4); + messageStoreConfig.setMaxHashSlotNum(100); + messageStoreConfig.setMaxIndexNum(100 * 10); + messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir") + File.separator + "store"); + messageStoreConfig.setHaListenPort(0); + MessageStore master = new RocksDBMessageStore(messageStoreConfig, null, new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>()); + + boolean load = master.load(); + assertTrue(load); + + try { + master.start(); + master.start(); + } finally { + master.shutdown(); + master.destroy(); + } + } + + @After + public void destroy() { + if (notExecuted()) { + return; + } + messageStore.shutdown(); + messageStore.destroy(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + File file = new File(messageStoreConfig.getStorePathRootDir()); + UtilAll.deleteFile(file); + } + + private MessageStore buildMessageStore() throws Exception { + return buildMessageStore(null, ""); + } + + private MessageStore buildMessageStore(String storePathRootDir, String topic) throws Exception { + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10); + messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10); + messageStoreConfig.setMaxHashSlotNum(10000); + messageStoreConfig.setMaxIndexNum(100 * 100); + messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH); + messageStoreConfig.setFlushIntervalConsumeQueue(1); + messageStoreConfig.setStoreType(storeType); + messageStoreConfig.setHaListenPort(0); + if (Strings.isNullOrEmpty(storePathRootDir)) { + UUID uuid = UUID.randomUUID(); + storePathRootDir = System.getProperty("java.io.tmpdir") + File.separator + "store-" + uuid.toString(); + } + messageStoreConfig.setStorePathRootDir(storePathRootDir); + ConcurrentMap topicConfigTable = new ConcurrentHashMap<>(); + topicConfigTable.put(topic, new TopicConfig(topic, 1, 1)); + return new RocksDBMessageStore(messageStoreConfig, + new BrokerStatsManager("simpleTest", true), + new MyMessageArrivingListener(), + new BrokerConfig(), topicConfigTable); + } + + @Test + public void testWriteAndRead() { + if (notExecuted()) { + return; + } + long ipv4HostMsgs = 10; + long ipv6HostMsgs = 10; + long totalMsgs = ipv4HostMsgs + ipv6HostMsgs; + queueTotal = 1; + messageBody = storeMessage.getBytes(); + for (long i = 0; i < ipv4HostMsgs; i++) { + messageStore.putMessage(buildMessage()); + } + + for (long i = 0; i < ipv6HostMsgs; i++) { + messageStore.putMessage(buildIPv6HostMessage()); + } + + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + for (long i = 0; i < totalMsgs; i++) { + GetMessageResult result = messageStore.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); + assertThat(result).isNotNull(); + result.release(); + } + verifyThatMasterIsFunctional(totalMsgs, messageStore); + } + + @Test + public void testLookMessageByOffset_OffsetIsFirst() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = new Random().nextInt(10); + String topic = "FooBar"; + int firstOffset = 0; + AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); + AppendMessageResult firstResult = appendMessageResultArray[0]; + + MessageExt messageExt = messageStore.lookMessageByOffset(firstResult.getWroteOffset()); + MessageExt messageExt1 = getDefaultMessageStore().lookMessageByOffset(firstResult.getWroteOffset(), firstResult.getWroteBytes()); + + assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset)); + assertThat(new String(messageExt1.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset)); + } + + @Test + public void testLookMessageByOffset_OffsetIsLast() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = new Random().nextInt(10); + String topic = "FooBar"; + AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); + int lastIndex = totalCount - 1; + AppendMessageResult lastResult = appendMessageResultArray[lastIndex]; + + MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastResult.getWroteOffset(), lastResult.getWroteBytes()); + + assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, lastIndex)); + } + + @Test + public void testLookMessageByOffset_OffsetIsOutOfBound() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = new Random().nextInt(10); + String topic = "FooBar"; + AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); + long lastOffset = getMaxOffset(appendMessageResultArray); + + MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastOffset); + + assertThat(messageExt).isNull(); + } + + @Test + public void testGetOffsetInQueueByTime() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); + for (AppendMessageResult appendMessageResult : appendMessageResults) { + long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp()); + CqUnit cqUnit = consumeQueue.get(offset); + assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset()); + assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes()); + } + } + + @Test + public void testGetOffsetInQueueByTime_TimestampIsSkewing() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + int skewing = 2; + + ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); + for (AppendMessageResult appendMessageResult : appendMessageResults) { + long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing); + CqUnit cqUnit = consumeQueue.get(offset); + assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset()); + assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes()); + } + } + + @Test + public void testGetOffsetInQueueByTime_TimestampSkewingIsLarge() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + int skewing = 20000; + + ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); + for (AppendMessageResult appendMessageResult : appendMessageResults) { + long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing); + CqUnit cqUnit = consumeQueue.get(offset); + assertThat(cqUnit.getPos()).isEqualTo(appendMessageResults[0].getWroteOffset()); + assertThat(cqUnit.getSize()).isEqualTo(appendMessageResults[0].getWroteBytes()); + } + } + + @Test + public void testGetOffsetInQueueByTime_ConsumeQueueNotFound1() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + int wrongQueueId = 1; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); + //Thread.sleep(10); + + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + long offset = messageStore.getOffsetInQueueByTime(topic, wrongQueueId, appendMessageResults[0].getStoreTimestamp()); + + assertThat(offset).isEqualTo(0); + } + + @Test + public void testGetOffsetInQueueByTime_ConsumeQueueNotFound2() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + int wrongQueueId = 1; + String topic = "FooBar"; + putMessages(totalCount, topic, queueId, false); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, 0); + + assertThat(messageStoreTimeStamp).isEqualTo(-1); + } + + @Test + public void testGetOffsetInQueueByTime_ConsumeQueueOffsetNotExist() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + int wrongQueueId = 1; + String topic = "FooBar"; + putMessages(totalCount, topic, queueId, true); + //Thread.sleep(10); + + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, -1); + + assertThat(messageStoreTimeStamp).isEqualTo(-1); + } + + @Test + public void testGetMessageStoreTimeStamp() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); + int minOffsetInQueue = (int) consumeQueue.getMinOffsetInQueue(); + for (int i = minOffsetInQueue; i < consumeQueue.getMaxOffsetInQueue(); i++) { + long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, queueId, i); + assertThat(messageStoreTimeStamp).isEqualTo(appendMessageResults[i].getStoreTimestamp()); + } + } + + @Test + public void testGetStoreTime_ParamIsNull() { + if (notExecuted()) { + return; + } + long storeTime = getStoreTime(null); + + assertThat(storeTime).isEqualTo(-1); + } + + @Test + public void testGetStoreTime_EverythingIsOk() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, queueId); + + for (int i = 0; i < totalCount; i++) { + CqUnit cqUnit = consumeQueue.get(i); + long storeTime = getStoreTime(cqUnit); + assertThat(storeTime).isEqualTo(appendMessageResults[i].getStoreTimestamp()); + } + } + + @Test + public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() { + if (notExecuted()) { + return; + } + long phyOffset = -10; + int size = 138; + CqUnit cqUnit = new CqUnit(0, phyOffset, size, 0); + long storeTime = getStoreTime(cqUnit); + + assertThat(storeTime).isEqualTo(-1); + } + + @Test + public void testPutMessage_whenMessagePropertyIsTooLong() { + if (notExecuted()) { + return; + } + String topicName = "messagePropertyIsTooLongTest"; + MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1); + assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); + assertEquals(0L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue()); + MessageExtBrokerInner normalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, 100); + assertEquals(messageStore.putMessage(normalMessage).getPutMessageStatus(), PutMessageStatus.PUT_OK); + assertEquals(1L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue()); + } + + private RocksDBMessageStore getDefaultMessageStore() { + return (RocksDBMessageStore) this.messageStore; + } + + private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId) { + return putMessages(totalCount, topic, queueId, false); + } + + private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId, boolean interval) { + AppendMessageResult[] appendMessageResultArray = new AppendMessageResult[totalCount]; + for (int i = 0; i < totalCount; i++) { + String messageBody = buildMessageBodyByOffset(storeMessage, i); + + MessageExtBrokerInner msgInner = + i < totalCount / 2 ? buildMessage(messageBody.getBytes(), topic) : buildIPv6HostMessage(messageBody.getBytes(), topic); + msgInner.setQueueId(queueId); + PutMessageResult result = messageStore.putMessage(msgInner); + appendMessageResultArray[i] = result.getAppendMessageResult(); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); + if (interval) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new RuntimeException("Thread sleep ERROR"); + } + } + } + return appendMessageResultArray; + } + + private long getMaxOffset(AppendMessageResult[] appendMessageResultArray) { + if (appendMessageResultArray == null) { + return 0; + } + AppendMessageResult last = appendMessageResultArray[appendMessageResultArray.length - 1]; + return last.getWroteOffset() + last.getWroteBytes(); + } + + private String buildMessageBodyByOffset(String message, long i) { + return String.format("%s offset %d", message, i); + } + + private long getStoreTime(CqUnit cqUnit) { + try { + Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass(); + Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class); + getStoreTime.setAccessible(true); + return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + + private MessageExtBrokerInner buildMessage(byte[] messageBody, String topic) { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(messageBody); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); + msg.setSysFlag(0); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(storeHost); + msg.setBornHost(bornHost); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + return msg; + } + + private MessageExtBrokerInner buildSpecifyLengthPropertyMessage(byte[] messageBody, String topic, int length) { + StringBuilder stringBuilder = new StringBuilder(); + Random random = new Random(); + for (int i = 0; i < length; i++) { + stringBuilder.append(random.nextInt(10)); + } + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.putUserProperty("test", stringBuilder.toString()); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(messageBody); + msg.setQueueId(0); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(storeHost); + msg.setBornHost(bornHost); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + return msg; + } + + private MessageExtBrokerInner buildIPv6HostMessage(byte[] messageBody, String topic) { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(messageBody); + msg.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); + msg.setSysFlag(0); + msg.setBornHostV6Flag(); + msg.setStoreHostAddressV6Flag(); + msg.setBornTimestamp(System.currentTimeMillis()); + try { + msg.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + + try { + msg.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + return msg; + } + + private MessageExtBrokerInner buildMessage() { + return buildMessage(messageBody, messageTopic); + } + + public MessageExtBatch buildMessageBatch(MessageBatch msgBatch) { + MessageExtBatch msgExtBatch = new MessageExtBatch(); + msgExtBatch.setTopic(messageTopic); + msgExtBatch.setTags("TAG1"); + msgExtBatch.setKeys("Hello"); + msgExtBatch.setBody(msgBatch.getBody()); + msgExtBatch.setKeys(String.valueOf(System.currentTimeMillis())); + msgExtBatch.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); + msgExtBatch.setSysFlag(0); + msgExtBatch.setBornTimestamp(System.currentTimeMillis()); + msgExtBatch.setStoreHost(storeHost); + msgExtBatch.setBornHost(bornHost); + return msgExtBatch; + } + + @Test + public void testGroupCommit() throws Exception { + if (notExecuted()) { + return; + } + long totalMsgs = 10; + queueTotal = 1; + messageBody = storeMessage.getBytes(); + for (long i = 0; i < totalMsgs; i++) { + messageStore.putMessage(buildMessage()); + } + + for (long i = 0; i < totalMsgs; i++) { + GetMessageResult result = messageStore.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + assertThat(result).isNotNull(); + result.release(); + } + verifyThatMasterIsFunctional(totalMsgs, messageStore); + } + + @Test + public void testMaxOffset() throws InterruptedException { + if (notExecuted()) { + return; + } + int firstBatchMessages = 3; + int queueId = 0; + messageBody = storeMessage.getBytes(); + + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(0); + + for (int i = 0; i < firstBatchMessages; i++) { + final MessageExtBrokerInner msg = buildMessage(); + msg.setQueueId(queueId); + messageStore.putMessage(msg); + } + + while (messageStore.dispatchBehindBytes() != 0) { + TimeUnit.MILLISECONDS.sleep(1); + } + + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages); + + // Disable the dispatcher + messageStore.getDispatcherList().clear(); + + int secondBatchMessages = 2; + + for (int i = 0; i < secondBatchMessages; i++) { + final MessageExtBrokerInner msg = buildMessage(); + msg.setQueueId(queueId); + messageStore.putMessage(msg); + } + + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages); + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, true)).isEqualTo(firstBatchMessages); + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, false)).isEqualTo(firstBatchMessages + secondBatchMessages); + } + + private MessageExtBrokerInner buildIPv6HostMessage() { + return buildIPv6HostMessage(messageBody, "FooBar"); + } + + private void verifyThatMasterIsFunctional(long totalMsgs, MessageStore master) { + for (long i = 0; i < totalMsgs; i++) { + master.putMessage(buildMessage()); + } + + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + for (long i = 0; i < totalMsgs; i++) { + GetMessageResult result = master.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); + assertThat(result).isNotNull(); + result.release(); + + } + } + + @Test + public void testPullSize() throws Exception { + if (notExecuted()) { + return; + } + String topic = "pullSizeTopic"; + + for (int i = 0; i < 32; i++) { + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + // wait for consume queue build + // the sleep time should be great than consume queue flush interval + //Thread.sleep(100); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + String group = "simple"; + GetMessageResult getMessageResult32 = messageStore.getMessage(group, topic, 0, 0, 32, null); + assertThat(getMessageResult32.getMessageBufferList().size()).isEqualTo(32); + getMessageResult32.release(); + + GetMessageResult getMessageResult20 = messageStore.getMessage(group, topic, 0, 0, 20, null); + assertThat(getMessageResult20.getMessageBufferList().size()).isEqualTo(20); + + getMessageResult20.release(); + GetMessageResult getMessageResult45 = messageStore.getMessage(group, topic, 0, 0, 10, null); + assertThat(getMessageResult45.getMessageBufferList().size()).isEqualTo(10); + getMessageResult45.release(); + + } + + @Test + public void testRecover() throws Exception { + if (notExecuted()) { + return; + } + String topic = "recoverTopic"; + messageBody = storeMessage.getBytes(); + for (int i = 0; i < 100; i++) { + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + + // Thread.sleep(100);//wait for build consumer queue + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + long maxPhyOffset = messageStore.getMaxPhyOffset(); + long maxCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); + + //1.just reboot + messageStore.shutdown(); + String storeRootDir = ((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir(); + messageStore = buildMessageStore(storeRootDir, topic); + boolean load = messageStore.load(); + assertTrue(load); + messageStore.start(); + assertTrue(maxPhyOffset == messageStore.getMaxPhyOffset()); + assertTrue(maxCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); + + //2.damage commit-log and reboot normal + for (int i = 0; i < 100; i++) { + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + //Thread.sleep(100); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + long secondLastPhyOffset = messageStore.getMaxPhyOffset(); + long secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); + + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + + + messageStore.shutdown(); + + //damage last message + damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset); + + //reboot + messageStore = buildMessageStore(storeRootDir, topic); + load = messageStore.load(); + assertTrue(load); + messageStore.start(); + assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset()); + assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); + + //3.damage commitlog and reboot abnormal + for (int i = 0; i < 100; i++) { + messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + //Thread.sleep(100); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + secondLastPhyOffset = messageStore.getMaxPhyOffset(); + secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); + + messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + messageStore.shutdown(); + + //damage last message + damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset); + //add abort file + String fileName = StorePathConfigHelper.getAbortFile(((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir()); + File file = new File(fileName); + UtilAll.ensureDirOK(file.getParent()); + file.createNewFile(); + + messageStore = buildMessageStore(storeRootDir, topic); + load = messageStore.load(); + assertTrue(load); + messageStore.start(); + assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset()); + assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); + + //message write again + for (int i = 0; i < 100; i++) { + messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + } + + @Test + public void testStorePathOK() { + if (notExecuted()) { + return; + } + if (messageStore instanceof RocksDBMessageStore) { + assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathPhysic())); + assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathLogic())); + } + } + + private boolean fileExists(String path) { + if (path != null) { + File f = new File(path); + return f.exists(); + } + return false; + } + + private void damageCommitLog(RocksDBMessageStore store, long offset) throws Exception { + assertThat(store).isNotNull(); + MessageStoreConfig messageStoreConfig = store.getMessageStoreConfig(); + File file = new File(messageStoreConfig.getStorePathCommitLog() + File.separator + "00000000000000000000"); + try (RandomAccessFile raf = new RandomAccessFile(file, "rw"); + FileChannel fileChannel = raf.getChannel()) { + MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, 1024 * 1024 * 10); + int bodyLen = mappedByteBuffer.getInt((int) offset + 84); + int topicLenIndex = (int) offset + 84 + bodyLen + 4; + mappedByteBuffer.position(topicLenIndex); + mappedByteBuffer.putInt(0); + mappedByteBuffer.putInt(0); + mappedByteBuffer.putInt(0); + mappedByteBuffer.putInt(0); + mappedByteBuffer.force(); + fileChannel.force(true); + } + } + + @Test + public void testPutMsgExceedsMaxLength() { + if (notExecuted()) { + return; + } + messageBody = new byte[4 * 1024 * 1024 + 1]; + MessageExtBrokerInner msg = buildMessage(); + + PutMessageResult result = messageStore.putMessage(msg); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.MESSAGE_ILLEGAL); + } + + @Test + public void testPutMsgBatchExceedsMaxLength() { + if (notExecuted()) { + return; + } + messageBody = new byte[4 * 1024 * 1024 + 1]; + MessageExtBrokerInner msg1 = buildMessage(); + MessageExtBrokerInner msg2 = buildMessage(); + MessageExtBrokerInner msg3 = buildMessage(); + + MessageBatch msgBatch = MessageBatch.generateFromList(Arrays.asList(msg1, msg2, msg3)); + msgBatch.setBody(msgBatch.encode()); + + MessageExtBatch msgExtBatch = buildMessageBatch(msgBatch); + + try { + PutMessageResult result = this.messageStore.putMessages(msgExtBatch); + } catch (Exception e) { + assertThat(e.getMessage()).contains("message body size exceeded"); + } + } + + @Test + public void testPutMsgWhenReplicasNotEnough() { + if (notExecuted()) { + return; + } + MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig(); + messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER); + messageStoreConfig.setTotalReplicas(2); + messageStoreConfig.setInSyncReplicas(2); + messageStoreConfig.setEnableAutoInSyncReplicas(false); + ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true); + this.messageStore.setAliveReplicaNumInGroup(1); + + MessageExtBrokerInner msg = buildMessage(); + PutMessageResult result = this.messageStore.putMessage(msg); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH); + ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false); + } + + @Test + public void testPutMsgWhenAdaptiveDegradation() { + if (notExecuted()) { + return; + } + MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig(); + messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER); + messageStoreConfig.setTotalReplicas(2); + messageStoreConfig.setInSyncReplicas(2); + messageStoreConfig.setEnableAutoInSyncReplicas(true); + ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true); + this.messageStore.setAliveReplicaNumInGroup(1); + + MessageExtBrokerInner msg = buildMessage(); + PutMessageResult result = this.messageStore.putMessage(msg); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); + ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false); + messageStoreConfig.setEnableAutoInSyncReplicas(false); + } + + @Test + public void testGetBulkCommitLogData() { + if (notExecuted()) { + return; + } + RocksDBMessageStore defaultMessageStore = (RocksDBMessageStore) messageStore; + + messageBody = new byte[2 * 1024 * 1024]; + + for (int i = 0; i < 10; i++) { + MessageExtBrokerInner msg1 = buildMessage(); + messageStore.putMessage(msg1); + } + + List bufferResultList = defaultMessageStore.getBulkCommitLogData(0, (int) defaultMessageStore.getMaxPhyOffset()); + List msgList = new ArrayList<>(); + for (SelectMappedBufferResult bufferResult : bufferResultList) { + msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false)); + bufferResult.release(); + } + + assertThat(msgList.size()).isEqualTo(10); + } + + @Test + public void testPutLongMessage() throws Exception { + if (notExecuted()) { + return; + } + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + CommitLog commitLog = ((RocksDBMessageStore) messageStore).getCommitLog(); + MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig(); + MessageExtEncoder.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get(); + + //body size, topic size, properties size exactly equal to max size + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); + messageExtBrokerInner.setTopic(new String(new byte[127])); + messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE])); + PutMessageResult encodeResult1 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult1 == null); + + //body size exactly more than max message body size + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]); + PutMessageResult encodeResult2 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult2.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + + //body size exactly equal to max message size + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 64 * 1024]); + PutMessageResult encodeResult3 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult3.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + + //message properties length more than properties maxSize + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); + messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE + 1])); + PutMessageResult encodeResult4 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult4.getPutMessageStatus() == PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); + + //message length more than buffer length capacity + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); + messageExtBrokerInner.setTopic(new String(new byte[Short.MAX_VALUE])); + messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE])); + PutMessageResult encodeResult5 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult5.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + } + + @Test + public void testDynamicMaxMessageSize() { + if (notExecuted()) { + return; + } + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig(); + int originMaxMessageSize = messageStoreConfig.getMaxMessageSize(); + + messageExtBrokerInner.setBody(new byte[originMaxMessageSize + 10]); + PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner); + assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + + int newMaxMessageSize = originMaxMessageSize + 10; + messageStoreConfig.setMaxMessageSize(newMaxMessageSize); + putMessageResult = messageStore.putMessage(messageExtBrokerInner); + assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK); + + messageStoreConfig.setMaxMessageSize(10); + putMessageResult = messageStore.putMessage(messageExtBrokerInner); + assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + + messageStoreConfig.setMaxMessageSize(originMaxMessageSize); + } + + @Test + public void testDeleteTopics() { + if (notExecuted()) { + return; + } + MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig(); + ConcurrentMap> consumeQueueTable = + ((RocksDBMessageStore) messageStore).getConsumeQueueTable(); + for (int i = 0; i < 10; i++) { + ConcurrentMap cqTable = new ConcurrentHashMap<>(); + String topicName = "topic-" + i; + for (int j = 0; j < 4; j++) { + ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(), + messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore); + cqTable.put(j, consumeQueue); + } + consumeQueueTable.put(topicName, cqTable); + } + Assert.assertEquals(consumeQueueTable.size(), 10); + HashSet resultSet = Sets.newHashSet("topic-3", "topic-5"); + messageStore.deleteTopics(Sets.difference(consumeQueueTable.keySet(), resultSet)); + Assert.assertEquals(consumeQueueTable.size(), 2); + Assert.assertEquals(resultSet, consumeQueueTable.keySet()); + } + + @Test + public void testCleanUnusedTopic() { + if (notExecuted()) { + return; + } + MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig(); + ConcurrentMap> consumeQueueTable = + ((RocksDBMessageStore) messageStore).getConsumeQueueTable(); + for (int i = 0; i < 10; i++) { + ConcurrentMap cqTable = new ConcurrentHashMap<>(); + String topicName = "topic-" + i; + for (int j = 0; j < 4; j++) { + ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(), + messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore); + cqTable.put(j, consumeQueue); + } + consumeQueueTable.put(topicName, cqTable); + } + Assert.assertEquals(consumeQueueTable.size(), 10); + HashSet resultSet = Sets.newHashSet("topic-3", "topic-5"); + messageStore.cleanUnusedTopic(resultSet); + Assert.assertEquals(consumeQueueTable.size(), 2); + Assert.assertEquals(resultSet, consumeQueueTable.keySet()); + } + + private class MyMessageArrivingListener implements MessageArrivingListener { + @Override + public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, + byte[] filterBitMap, Map properties) { + } + } + + private boolean notExecuted() { + return MixAll.isMac(); + } +} + + diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java index b2d99c3ed..17a2b5e19 100644 --- a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java +++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java @@ -38,11 +38,16 @@ public class StoreTestUtil { public static boolean isCommitLogAvailable(DefaultMessageStore store) { try { + Field serviceField = null; + if (store instanceof RocksDBMessageStore) { + serviceField = store.getClass().getSuperclass().getDeclaredField("reputMessageService"); + } else { + serviceField = store.getClass().getDeclaredField("reputMessageService"); + } - Field serviceField = store.getClass().getDeclaredField("reputMessageService"); serviceField.setAccessible(true); DefaultMessageStore.ReputMessageService reputService = - (DefaultMessageStore.ReputMessageService) serviceField.get(store); + (DefaultMessageStore.ReputMessageService) serviceField.get(store); Method method = DefaultMessageStore.ReputMessageService.class.getDeclaredMethod("isCommitLogAvailable"); method.setAccessible(true); diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java index 54174ac16..fa8f41dbf 100644 --- a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java @@ -36,7 +36,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; - +import org.rocksdb.RocksDBException; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.anyLong; @@ -114,7 +114,7 @@ public class HAServerTest { } @Test - public void inSyncReplicasNums() throws IOException { + public void inSyncReplicasNums() throws IOException, RocksDBException { DefaultMessageStore messageStore = mockMessageStore(); doReturn(123L).when(messageStore).getMaxPhyOffset(); doReturn(123L).when(messageStore).getMasterFlushedOffset(); @@ -150,7 +150,7 @@ public class HAServerTest { } @Test - public void isSlaveOK() throws IOException { + public void isSlaveOK() throws IOException, RocksDBException { DefaultMessageStore messageStore = mockMessageStore(); doReturn(123L).when(messageStore).getMaxPhyOffset(); doReturn(123L).when(messageStore).getMasterFlushedOffset(); @@ -175,7 +175,8 @@ public class HAServerTest { } @Test - public void putRequest_SingleAck() throws IOException, ExecutionException, InterruptedException, TimeoutException { + public void putRequest_SingleAck() + throws IOException, ExecutionException, InterruptedException, TimeoutException, RocksDBException { CommitLog.GroupCommitRequest request = new CommitLog.GroupCommitRequest(124, 4000, 1); this.haService.putRequest(request); @@ -192,7 +193,8 @@ public class HAServerTest { } @Test - public void putRequest_MultipleAckAndRequests() throws IOException, ExecutionException, InterruptedException { + public void putRequest_MultipleAckAndRequests() + throws IOException, ExecutionException, InterruptedException, RocksDBException { CommitLog.GroupCommitRequest oneAck = new CommitLog.GroupCommitRequest(124, 4000, 2); this.haService.putRequest(oneAck); @@ -218,7 +220,7 @@ public class HAServerTest { } @Test - public void getPush2SlaveMaxOffset() throws IOException { + public void getPush2SlaveMaxOffset() throws IOException, RocksDBException { DefaultMessageStore messageStore = mockMessageStore(); doReturn(123L).when(messageStore).getMaxPhyOffset(); doReturn(123L).when(messageStore).getMasterFlushedOffset(); @@ -256,7 +258,7 @@ public class HAServerTest { this.haClientList.add(haClient); } - private DefaultMessageStore mockMessageStore() throws IOException { + private DefaultMessageStore mockMessageStore() throws IOException, RocksDBException { DefaultMessageStore messageStore = mock(DefaultMessageStore.class); BrokerConfig brokerConfig = mock(BrokerConfig.class); diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java index 27dcff141..db5c5af4c 100644 --- a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java +++ b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java @@ -41,6 +41,7 @@ import org.junit.Assert; import org.junit.Assume; import org.junit.Ignore; import org.junit.Test; +import org.rocksdb.RocksDBException; import java.io.File; import java.net.InetAddress; @@ -180,7 +181,7 @@ public class AutoSwitchHATest { private boolean changeMasterAndPutMessage(DefaultMessageStore master, MessageStoreConfig masterConfig, DefaultMessageStore slave, long slaveId, MessageStoreConfig slaveConfig, int epoch, String masterHaAddress, - int totalPutMessageNums) { + int totalPutMessageNums) throws RocksDBException { boolean flag = true; // Change role diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java index d7d13d61e..edaa5d19f 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java @@ -16,17 +16,14 @@ */ package org.apache.rocketmq.tieredstore; -import com.google.common.base.Stopwatch; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.sdk.metrics.InstrumentSelector; -import io.opentelemetry.sdk.metrics.ViewBuilder; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; + +import com.google.common.base.Stopwatch; + import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.MixAll; @@ -55,6 +52,12 @@ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant; import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.ViewBuilder; + public class TieredMessageStore extends AbstractPluginMessageStore { protected static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java index 2a7d3fba4..1ecb1fa2c 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java @@ -106,8 +106,8 @@ public class ExportMetadataInRocksDBCommand implements SubCommand { final Map jsonConfig = new HashMap<>(); final Map configTable = new HashMap<>(); iterateKvStore(kvStore, (key, value) -> { - final String configKey = new String(key, DataConverter.charset); - final String configValue = new String(value, DataConverter.charset); + final String configKey = new String(key, DataConverter.CHARSET_UTF8); + final String configValue = new String(value, DataConverter.CHARSET_UTF8); final JSONObject jsonObject = JSONObject.parseObject(configValue); configTable.put(configKey, jsonObject); } @@ -120,8 +120,8 @@ public class ExportMetadataInRocksDBCommand implements SubCommand { } else { AtomicLong count = new AtomicLong(0); iterateKvStore(kvStore, (key, value) -> { - final String configKey = new String(key, DataConverter.charset); - final String configValue = new String(value, DataConverter.charset); + final String configKey = new String(key, DataConverter.CHARSET_UTF8); + final String configValue = new String(value, DataConverter.CHARSET_UTF8); System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), configKey, configValue); }); } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java new file mode 100644 index 000000000..b987ad873 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.metadata; + +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.config.RocksDBConfigManager; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +public class RocksDBConfigToJsonCommand implements SubCommand { + private static final String TOPICS_JSON_CONFIG = "topics"; + private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; + + @Override + public String commandName() { + return "rocksDBConfigToJson"; + } + + @Override + public String commandDesc() { + return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option pathOption = new Option("p", "path", true, + "Absolute path to the metadata directory"); + pathOption.setRequired(true); + options.addOption(pathOption); + + Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + + "topics/subscriptionGroups"); + configTypeOption.setRequired(true); + options.addOption(configTypeOption); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { + String path = commandLine.getOptionValue("path").trim(); + if (StringUtils.isEmpty(path) || !new File(path).exists()) { + System.out.print("Rocksdb path is invalid.\n"); + return; + } + + String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); + + final long memTableFlushInterval = 60 * 60 * 1000L; + RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); + try { + if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { + // for topics.json + final Map topicsJsonConfig = new HashMap<>(); + final Map topicConfigTable = new HashMap<>(); + boolean isLoad = kvConfigManager.load(path, (key, value) -> { + final String topic = new String(key, DataConverter.CHARSET_UTF8); + final String topicConfig = new String(value, DataConverter.CHARSET_UTF8); + final JSONObject jsonObject = JSONObject.parseObject(topicConfig); + topicConfigTable.put(topic, jsonObject); + }); + + if (isLoad) { + topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); + final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); + System.out.print(topicsJsonStr + "\n"); + return; + } + } + if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { + // for subscriptionGroup.json + final Map subscriptionGroupJsonConfig = new HashMap<>(); + final Map subscriptionGroupTable = new HashMap<>(); + boolean isLoad = kvConfigManager.load(path, (key, value) -> { + final String subscriptionGroup = new String(key, DataConverter.CHARSET_UTF8); + final String subscriptionGroupConfig = new String(value, DataConverter.CHARSET_UTF8); + final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); + subscriptionGroupTable.put(subscriptionGroup, jsonObject); + }); + + if (isLoad) { + subscriptionGroupJsonConfig.put("subscriptionGroupTable", + (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); + final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); + System.out.print(subscriptionGroupJsonStr + "\n"); + return; + } + } + System.out.print("Config type was not recognized, configType=" + configType + "\n"); + } finally { + kvConfigManager.stop(); + } + } +} -- 2.32.0.windows.2