29481 lines
1.5 MiB
Diff
29481 lines
1.5 MiB
Diff
|
From 0a046b85611bdc93c529650dc951f4cabbb61db9 Mon Sep 17 00:00:00 2001
|
|||
|
|
From: zhiliatom <zhiliatox@163.com>
|
|||
|
|
Date: Fri, 8 Dec 2023 09:25:41 +0800
|
|||
|
|
Subject: [PATCH] [ISSUE #7064] [RIP-66-2] Support KV(RocksDB) Storage for
|
|||
|
|
ConsumeQueue
|
|||
|
|
|
|||
|
|
---
|
|||
|
|
.../rocketmq/broker/BrokerController.java | 74 +-
|
|||
|
|
.../broker/controller/ReplicasManager.java | 4 +-
|
|||
|
|
.../offset/RocksDBConsumerOffsetManager.java | 8 +-
|
|||
|
|
.../broker/processor/AckMessageProcessor.java | 2 +-
|
|||
|
|
.../processor/AdminBrokerProcessor.java | 28 +-
|
|||
|
|
.../ChangeInvisibleTimeProcessor.java | 4 +-
|
|||
|
|
.../processor/PopBufferMergeService.java | 4 +-
|
|||
|
|
.../broker/processor/PopMessageProcessor.java | 2 +-
|
|||
|
|
.../broker/processor/PopReviveService.java | 6 +-
|
|||
|
|
.../RocksDBSubscriptionGroupManager.java | 10 +-
|
|||
|
|
.../topic/RocksDBTopicConfigManager.java | 8 +-
|
|||
|
|
.../processor/PopReviveServiceTest.java | 4 +-
|
|||
|
|
common/pom.xml | 2 +-
|
|||
|
|
.../org/apache/rocketmq/common/MixAll.java | 1 +
|
|||
|
|
.../rocketmq/common/attribute/CQType.java | 3 +-
|
|||
|
|
.../common/config/AbstractRocksDBStorage.java | 112 +-
|
|||
|
|
.../common/config/ConfigRocksDBStorage.java | 2 +-
|
|||
|
|
.../common/message/MessageExtBrokerInner.java | 13 +
|
|||
|
|
.../rocketmq/common/topic/TopicValidator.java | 2 +
|
|||
|
|
.../rocketmq/common/utils/DataConverter.java | 2 +-
|
|||
|
|
.../rocketmq/example/quickstart/Producer.java | 2 +-
|
|||
|
|
metrics | 241 ++
|
|||
|
|
patch009-modify-CRLF-to-LF.patch | 127 +
|
|||
|
|
patch010-backport-add-some-fixes | 1286 ++++++
|
|||
|
|
patch011-backport-optimize-config | 1390 +++++++
|
|||
|
|
...optimize-opentelemetry-metric-config.patch | 2081 ++++++++++
|
|||
|
|
...-backport-enhance-rockdbconfigtojson.patch | 2920 +++++++++++++
|
|||
|
|
patch013-backport-enhance-admin-output.patch | 892 ++++
|
|||
|
|
...ueue-Selection-Strategy-Optimization.patch | 2023 +++++++++
|
|||
|
|
patch015-backport-fix-some-bugs.patch | 1894 +++++++++
|
|||
|
|
...rt-Optimize-fault-tolerant-mechanism.patch | 520 +++
|
|||
|
|
...port-Convergent-thread-pool-creation.patch | 2243 ++++++++++
|
|||
|
|
...ckport-enhancement-of-tiered-storage.patch | 601 +++
|
|||
|
|
patch019-backport-some-bugfix.patch | 1499 +++++++
|
|||
|
|
patch020-backport-add-goaway-mechanism.patch | 3696 +++++++++++++++++
|
|||
|
|
patch021-backport-some-enhancements.patch | 344 ++
|
|||
|
|
pom.xml | 4 +-
|
|||
|
|
store/pom.xml | 4 +
|
|||
|
|
.../org/apache/rocketmq/store/CommitLog.java | 94 +-
|
|||
|
|
.../rocketmq/store/CommitLogDispatcher.java | 5 +-
|
|||
|
|
.../apache/rocketmq/store/ConsumeQueue.java | 93 +-
|
|||
|
|
.../rocketmq/store/DefaultMessageStore.java | 279 +-
|
|||
|
|
.../apache/rocketmq/store/MessageStore.java | 54 +-
|
|||
|
|
.../rocketmq/store/RocksDBMessageStore.java | 169 +
|
|||
|
|
.../apache/rocketmq/store/RunningFlags.java | 22 +-
|
|||
|
|
.../store/config/MessageStoreConfig.java | 33 +-
|
|||
|
|
.../store/dledger/DLedgerCommitLog.java | 53 +-
|
|||
|
|
.../apache/rocketmq/store/ha/HAService.java | 3 +-
|
|||
|
|
.../ha/autoswitch/AutoSwitchHAClient.java | 2 +-
|
|||
|
|
.../ha/autoswitch/AutoSwitchHAService.java | 7 +-
|
|||
|
|
.../plugin/AbstractPluginMessageStore.java | 42 +-
|
|||
|
|
.../queue/AbstractConsumeQueueStore.java | 105 +
|
|||
|
|
.../store/queue/BatchConsumeQueue.java | 20 +
|
|||
|
|
.../store/queue/ConsumeQueueInterface.java | 27 +-
|
|||
|
|
.../store/queue/ConsumeQueueStore.java | 293 +-
|
|||
|
|
.../queue/ConsumeQueueStoreInterface.java | 289 ++
|
|||
|
|
.../rocketmq/store/queue/MultiDispatch.java | 76 +
|
|||
|
|
.../store/queue/QueueOffsetOperator.java | 8 +
|
|||
|
|
.../store/queue/RocksDBConsumeQueue.java | 437 ++
|
|||
|
|
.../queue/RocksDBConsumeQueueOffsetTable.java | 641 +++
|
|||
|
|
.../store/queue/RocksDBConsumeQueueStore.java | 441 ++
|
|||
|
|
.../store/queue/RocksDBConsumeQueueTable.java | 312 ++
|
|||
|
|
.../ConsumeQueueCompactionFilterFactory.java | 47 +
|
|||
|
|
.../rocksdb/ConsumeQueueRocksDBStorage.java | 133 +
|
|||
|
|
.../store/rocksdb/RocksDBOptionsFactory.java | 161 +
|
|||
|
|
.../store/timer/TimerMessageStore.java | 65 +-
|
|||
|
|
.../store/DefaultMessageStoreTest.java | 5 +-
|
|||
|
|
.../rocketmq/store/MultiDispatchTest.java | 8 +-
|
|||
|
|
.../store/RocksDBMessageStoreTest.java | 1060 +++++
|
|||
|
|
.../apache/rocketmq/store/StoreTestUtil.java | 9 +-
|
|||
|
|
.../rocketmq/store/ha/HAServerTest.java | 16 +-
|
|||
|
|
.../store/ha/autoswitch/AutoSwitchHATest.java | 3 +-
|
|||
|
|
.../tieredstore/TieredMessageStore.java | 15 +-
|
|||
|
|
.../ExportMetadataInRocksDBCommand.java | 8 +-
|
|||
|
|
.../metadata/RocksDBConfigToJsonCommand.java | 118 +
|
|||
|
|
75 files changed, 26543 insertions(+), 668 deletions(-)
|
|||
|
|
create mode 100644 metrics
|
|||
|
|
create mode 100644 patch009-modify-CRLF-to-LF.patch
|
|||
|
|
create mode 100644 patch010-backport-add-some-fixes
|
|||
|
|
create mode 100644 patch011-backport-optimize-config
|
|||
|
|
create mode 100644 patch011-backport-optimize-opentelemetry-metric-config.patch
|
|||
|
|
create mode 100644 patch012-backport-enhance-rockdbconfigtojson.patch
|
|||
|
|
create mode 100644 patch013-backport-enhance-admin-output.patch
|
|||
|
|
create mode 100644 patch014-backport-Queue-Selection-Strategy-Optimization.patch
|
|||
|
|
create mode 100644 patch015-backport-fix-some-bugs.patch
|
|||
|
|
create mode 100644 patch016-backport-Optimize-fault-tolerant-mechanism.patch
|
|||
|
|
create mode 100644 patch017-backport-Convergent-thread-pool-creation.patch
|
|||
|
|
create mode 100644 patch018-backport-enhancement-of-tiered-storage.patch
|
|||
|
|
create mode 100644 patch019-backport-some-bugfix.patch
|
|||
|
|
create mode 100644 patch020-backport-add-goaway-mechanism.patch
|
|||
|
|
create mode 100644 patch021-backport-some-enhancements.patch
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java
|
|||
|
|
create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java
|
|||
|
|
create mode 100644 store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java
|
|||
|
|
create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
index d4bded600..9f1fd0ad0 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
@@ -16,7 +16,33 @@
|
|||
|
|
*/
|
|||
|
|
package org.apache.rocketmq.broker;
|
|||
|
|
|
|||
|
|
+import java.io.IOException;
|
|||
|
|
+import java.net.InetSocketAddress;
|
|||
|
|
+import java.util.AbstractMap;
|
|||
|
|
+import java.util.ArrayList;
|
|||
|
|
+import java.util.Arrays;
|
|||
|
|
+import java.util.Collections;
|
|||
|
|
+import java.util.HashMap;
|
|||
|
|
+import java.util.List;
|
|||
|
|
+import java.util.Map;
|
|||
|
|
+import java.util.Objects;
|
|||
|
|
+import java.util.Optional;
|
|||
|
|
+import java.util.concurrent.BlockingQueue;
|
|||
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+import java.util.concurrent.CountDownLatch;
|
|||
|
|
+import java.util.concurrent.ExecutorService;
|
|||
|
|
+import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+import java.util.concurrent.ScheduledFuture;
|
|||
|
|
+import java.util.concurrent.TimeUnit;
|
|||
|
|
+import java.util.concurrent.locks.Lock;
|
|||
|
|
+import java.util.concurrent.locks.ReentrantLock;
|
|||
|
|
+import java.util.function.Function;
|
|||
|
|
+import java.util.stream.Collectors;
|
|||
|
|
+
|
|||
|
|
import com.google.common.collect.Lists;
|
|||
|
|
+
|
|||
|
|
import org.apache.rocketmq.acl.AccessValidator;
|
|||
|
|
import org.apache.rocketmq.acl.plain.PlainAccessValidator;
|
|||
|
|
import org.apache.rocketmq.broker.client.ClientHousekeepingService;
|
|||
|
|
@@ -126,7 +152,7 @@ import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
import org.apache.rocketmq.store.MessageArrivingListener;
|
|||
|
|
import org.apache.rocketmq.store.MessageStore;
|
|||
|
|
import org.apache.rocketmq.store.PutMessageResult;
|
|||
|
|
-import org.apache.rocketmq.store.StoreType;
|
|||
|
|
+import org.apache.rocketmq.store.RocksDBMessageStore;
|
|||
|
|
import org.apache.rocketmq.store.config.BrokerRole;
|
|||
|
|
import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
|
|||
|
|
@@ -141,31 +167,6 @@ import org.apache.rocketmq.store.timer.TimerCheckpoint;
|
|||
|
|
import org.apache.rocketmq.store.timer.TimerMessageStore;
|
|||
|
|
import org.apache.rocketmq.store.timer.TimerMetrics;
|
|||
|
|
|
|||
|
|
-import java.io.IOException;
|
|||
|
|
-import java.net.InetSocketAddress;
|
|||
|
|
-import java.util.AbstractMap;
|
|||
|
|
-import java.util.ArrayList;
|
|||
|
|
-import java.util.Arrays;
|
|||
|
|
-import java.util.Collections;
|
|||
|
|
-import java.util.HashMap;
|
|||
|
|
-import java.util.List;
|
|||
|
|
-import java.util.Map;
|
|||
|
|
-import java.util.Objects;
|
|||
|
|
-import java.util.Optional;
|
|||
|
|
-import java.util.concurrent.BlockingQueue;
|
|||
|
|
-import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
-import java.util.concurrent.ConcurrentMap;
|
|||
|
|
-import java.util.concurrent.CountDownLatch;
|
|||
|
|
-import java.util.concurrent.ExecutorService;
|
|||
|
|
-import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
-import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
-import java.util.concurrent.ScheduledFuture;
|
|||
|
|
-import java.util.concurrent.TimeUnit;
|
|||
|
|
-import java.util.concurrent.locks.Lock;
|
|||
|
|
-import java.util.concurrent.locks.ReentrantLock;
|
|||
|
|
-import java.util.function.Function;
|
|||
|
|
-import java.util.stream.Collectors;
|
|||
|
|
-
|
|||
|
|
public class BrokerController {
|
|||
|
|
protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
|
|||
|
|
private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME);
|
|||
|
|
@@ -308,7 +309,7 @@ public class BrokerController {
|
|||
|
|
this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort()));
|
|||
|
|
this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat());
|
|||
|
|
this.broadcastOffsetManager = new BroadcastOffsetManager(this);
|
|||
|
|
- if (isEnableRocksDBStore()) {
|
|||
|
|
+ if (this.messageStoreConfig.isEnableRocksDBStore()) {
|
|||
|
|
this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this);
|
|||
|
|
this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this);
|
|||
|
|
this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqConsumerOffsetManager(this) : new RocksDBConsumerOffsetManager(this);
|
|||
|
|
@@ -747,7 +748,12 @@ public class BrokerController {
|
|||
|
|
public boolean initializeMessageStore() {
|
|||
|
|
boolean result = true;
|
|||
|
|
try {
|
|||
|
|
- DefaultMessageStore defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable());
|
|||
|
|
+ DefaultMessageStore defaultMessageStore;
|
|||
|
|
+ if (this.messageStoreConfig.isEnableRocksDBStore()) {
|
|||
|
|
+ defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable());
|
|||
|
|
+ } else {
|
|||
|
|
+ defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable());
|
|||
|
|
+ }
|
|||
|
|
|
|||
|
|
if (messageStoreConfig.isEnableDLegerCommitLog()) {
|
|||
|
|
DLedgerRoleChangeHandler roleChangeHandler =
|
|||
|
|
@@ -944,16 +950,16 @@ public class BrokerController {
|
|||
|
|
this.transactionalMessageService = ServiceProvider.loadClass(TransactionalMessageService.class);
|
|||
|
|
if (null == this.transactionalMessageService) {
|
|||
|
|
this.transactionalMessageService = new TransactionalMessageServiceImpl(
|
|||
|
|
- new TransactionalMessageBridge(this, this.getMessageStore()));
|
|||
|
|
+ new TransactionalMessageBridge(this, this.getMessageStore()));
|
|||
|
|
LOG.warn("Load default transaction message hook service: {}",
|
|||
|
|
- TransactionalMessageServiceImpl.class.getSimpleName());
|
|||
|
|
+ TransactionalMessageServiceImpl.class.getSimpleName());
|
|||
|
|
}
|
|||
|
|
this.transactionalMessageCheckListener = ServiceProvider.loadClass(
|
|||
|
|
- AbstractTransactionalMessageCheckListener.class);
|
|||
|
|
+ AbstractTransactionalMessageCheckListener.class);
|
|||
|
|
if (null == this.transactionalMessageCheckListener) {
|
|||
|
|
this.transactionalMessageCheckListener = new DefaultTransactionalMessageCheckListener();
|
|||
|
|
LOG.warn("Load default discard message hook service: {}",
|
|||
|
|
- DefaultTransactionalMessageCheckListener.class.getSimpleName());
|
|||
|
|
+ DefaultTransactionalMessageCheckListener.class.getSimpleName());
|
|||
|
|
}
|
|||
|
|
this.transactionalMessageCheckListener.setBrokerController(this);
|
|||
|
|
this.transactionalMessageCheckService = new TransactionalMessageCheckService(this);
|
|||
|
|
@@ -2412,8 +2418,4 @@ public class BrokerController {
|
|||
|
|
public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) {
|
|||
|
|
this.coldDataCgCtrService = coldDataCgCtrService;
|
|||
|
|
}
|
|||
|
|
-
|
|||
|
|
- public boolean isEnableRocksDBStore() {
|
|||
|
|
- return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.messageStoreConfig.getStoreType());
|
|||
|
|
- }
|
|||
|
|
}
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
index a989e6e68..a1d711cb2 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
@@ -224,7 +224,7 @@ public class ReplicasManager {
|
|||
|
|
|
|||
|
|
public synchronized void changeBrokerRole(final Long newMasterBrokerId, final String newMasterAddress,
|
|||
|
|
final Integer newMasterEpoch,
|
|||
|
|
- final Integer syncStateSetEpoch, final Set<Long> syncStateSet) {
|
|||
|
|
+ final Integer syncStateSetEpoch, final Set<Long> syncStateSet) throws Exception {
|
|||
|
|
if (newMasterBrokerId != null && newMasterEpoch > this.masterEpoch) {
|
|||
|
|
if (newMasterBrokerId.equals(this.brokerControllerId)) {
|
|||
|
|
changeToMaster(newMasterEpoch, syncStateSetEpoch, syncStateSet);
|
|||
|
|
@@ -234,7 +234,7 @@ public class ReplicasManager {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set<Long> syncStateSet) {
|
|||
|
|
+ public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set<Long> syncStateSet) throws Exception {
|
|||
|
|
synchronized (this) {
|
|||
|
|
if (newMasterEpoch > this.masterEpoch) {
|
|||
|
|
LOGGER.info("Begin to change to master, brokerName:{}, replicas:{}, new Epoch:{}", this.brokerConfig.getBrokerName(), this.brokerAddress, newMasterEpoch);
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java
|
|||
|
|
index 5695a3356..05b53b0bc 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java
|
|||
|
|
@@ -33,7 +33,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager {
|
|||
|
|
|
|||
|
|
public RocksDBConsumerOffsetManager(BrokerController brokerController) {
|
|||
|
|
super(brokerController);
|
|||
|
|
- this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval());
|
|||
|
|
+ this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs());
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
@@ -49,7 +49,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager {
|
|||
|
|
@Override
|
|||
|
|
protected void removeConsumerOffset(String topicAtGroup) {
|
|||
|
|
try {
|
|||
|
|
- byte[] keyBytes = topicAtGroup.getBytes(DataConverter.charset);
|
|||
|
|
+ byte[] keyBytes = topicAtGroup.getBytes(DataConverter.CHARSET_UTF8);
|
|||
|
|
this.rocksDBConfigManager.delete(keyBytes);
|
|||
|
|
} catch (Exception e) {
|
|||
|
|
LOG.error("kv remove consumerOffset Failed, {}", topicAtGroup);
|
|||
|
|
@@ -58,7 +58,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager {
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
protected void decode0(final byte[] key, final byte[] body) {
|
|||
|
|
- String topicAtGroup = new String(key, DataConverter.charset);
|
|||
|
|
+ String topicAtGroup = new String(key, DataConverter.CHARSET_UTF8);
|
|||
|
|
RocksDBOffsetSerializeWrapper wrapper = JSON.parseObject(body, RocksDBOffsetSerializeWrapper.class);
|
|||
|
|
|
|||
|
|
this.offsetTable.put(topicAtGroup, wrapper.getOffsetTable());
|
|||
|
|
@@ -93,7 +93,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupName, final ConcurrentMap<Integer, Long> offsetMap) throws Exception {
|
|||
|
|
- byte[] keyBytes = topicGroupName.getBytes(DataConverter.charset);
|
|||
|
|
+ byte[] keyBytes = topicGroupName.getBytes(DataConverter.CHARSET_UTF8);
|
|||
|
|
RocksDBOffsetSerializeWrapper wrapper = new RocksDBOffsetSerializeWrapper();
|
|||
|
|
wrapper.setOffsetTable(offsetMap);
|
|||
|
|
byte[] valueBytes = JSON.toJSONBytes(wrapper, SerializerFeature.BrowserCompatible);
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
|
|||
|
|
index 244b459d6..59a3e63b2 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
|
|||
|
|
@@ -253,7 +253,7 @@ public class AckMessageProcessor implements NettyRequestProcessor {
|
|||
|
|
|
|||
|
|
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
|
|||
|
|
msgInner.setTopic(reviveTopic);
|
|||
|
|
- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset));
|
|||
|
|
+ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
msgInner.setQueueId(rqId);
|
|||
|
|
if (ackMsg instanceof BatchAckMsg) {
|
|||
|
|
msgInner.setTags(PopAckConstants.BATCH_ACK_TAG);
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
index e77120e15..dd4ec960f 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
@@ -539,14 +539,18 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
|
|||
|
|
final Set<String> groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic);
|
|||
|
|
// delete pop retry topics first
|
|||
|
|
- for (String group : groups) {
|
|||
|
|
- final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group);
|
|||
|
|
- if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) {
|
|||
|
|
- deleteTopicInBroker(popRetryTopic);
|
|||
|
|
+ try {
|
|||
|
|
+ for (String group : groups) {
|
|||
|
|
+ final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group);
|
|||
|
|
+ if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) {
|
|||
|
|
+ deleteTopicInBroker(popRetryTopic);
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
+ // delete topic
|
|||
|
|
+ deleteTopicInBroker(topic);
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
|
|||
|
|
}
|
|||
|
|
- // delete topic
|
|||
|
|
- deleteTopicInBroker(topic);
|
|||
|
|
response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
response.setRemark(null);
|
|||
|
|
return response;
|
|||
|
|
@@ -2081,7 +2085,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
public RemotingCommand cleanExpiredConsumeQueue() {
|
|||
|
|
LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: start.");
|
|||
|
|
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
- brokerController.getMessageStore().cleanExpiredConsumerQueue();
|
|||
|
|
+ try {
|
|||
|
|
+ brokerController.getMessageStore().cleanExpiredConsumerQueue();
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
|
|||
|
|
+ }
|
|||
|
|
LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: end.");
|
|||
|
|
response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
response.setRemark(null);
|
|||
|
|
@@ -2781,7 +2789,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
|
|||
|
|
final ReplicasManager replicasManager = this.brokerController.getReplicasManager();
|
|||
|
|
if (replicasManager != null) {
|
|||
|
|
- replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet());
|
|||
|
|
+ try {
|
|||
|
|
+ replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet());
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ throw new RemotingCommandException(e.getMessage());
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
response.setRemark(null);
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
|
|||
|
|
index 2ccdf07f6..bdfffff09 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
|
|||
|
|
@@ -180,7 +180,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
msgInner.setTopic(reviveTopic);
|
|||
|
|
- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset));
|
|||
|
|
+ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
msgInner.setQueueId(rqId);
|
|||
|
|
msgInner.setTags(PopAckConstants.ACK_TAG);
|
|||
|
|
msgInner.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
@@ -216,7 +216,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor {
|
|||
|
|
ck.addDiff(0);
|
|||
|
|
ck.setBrokerName(brokerName);
|
|||
|
|
|
|||
|
|
- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset));
|
|||
|
|
+ msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
msgInner.setQueueId(reviveQid);
|
|||
|
|
msgInner.setTags(PopAckConstants.CK_TAG);
|
|||
|
|
msgInner.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java
|
|||
|
|
index b7ba8ad4a..8a85dd8fe 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java
|
|||
|
|
@@ -633,7 +633,7 @@ public class PopBufferMergeService extends ServiceThread {
|
|||
|
|
ackMsg.setQueueId(point.getQueueId());
|
|||
|
|
ackMsg.setPopTime(point.getPopTime());
|
|||
|
|
msgInner.setTopic(popMessageProcessor.reviveTopic);
|
|||
|
|
- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset));
|
|||
|
|
+ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
msgInner.setQueueId(pointWrapper.getReviveQueueId());
|
|||
|
|
msgInner.setTags(PopAckConstants.ACK_TAG);
|
|||
|
|
msgInner.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
@@ -673,7 +673,7 @@ public class PopBufferMergeService extends ServiceThread {
|
|||
|
|
batchAckMsg.setQueueId(point.getQueueId());
|
|||
|
|
batchAckMsg.setPopTime(point.getPopTime());
|
|||
|
|
msgInner.setTopic(popMessageProcessor.reviveTopic);
|
|||
|
|
- msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.charset));
|
|||
|
|
+ msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
msgInner.setQueueId(pointWrapper.getReviveQueueId());
|
|||
|
|
msgInner.setTags(PopAckConstants.BATCH_ACK_TAG);
|
|||
|
|
msgInner.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
|
|||
|
|
index 0d9bdf143..f5d07c5aa 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
|
|||
|
|
@@ -685,7 +685,7 @@ public class PopMessageProcessor implements NettyRequestProcessor {
|
|||
|
|
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
|
|||
|
|
|
|||
|
|
msgInner.setTopic(reviveTopic);
|
|||
|
|
- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset));
|
|||
|
|
+ msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
msgInner.setQueueId(reviveQid);
|
|||
|
|
msgInner.setTags(PopAckConstants.CK_TAG);
|
|||
|
|
msgInner.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
|
|||
|
|
index d5174d3d1..4f80752e1 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
|
|||
|
|
@@ -356,7 +356,7 @@ public class PopReviveService extends ServiceThread {
|
|||
|
|
}
|
|||
|
|
for (MessageExt messageExt : messageExts) {
|
|||
|
|
if (PopAckConstants.CK_TAG.equals(messageExt.getTags())) {
|
|||
|
|
- String raw = new String(messageExt.getBody(), DataConverter.charset);
|
|||
|
|
+ String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);
|
|||
|
|
if (brokerController.getBrokerConfig().isEnablePopLog()) {
|
|||
|
|
POP_LOGGER.info("reviveQueueId={},find ck, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
|
|||
|
|
}
|
|||
|
|
@@ -371,7 +371,7 @@ public class PopReviveService extends ServiceThread {
|
|||
|
|
firstRt = point.getReviveTime();
|
|||
|
|
}
|
|||
|
|
} else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) {
|
|||
|
|
- String raw = new String(messageExt.getBody(), DataConverter.charset);
|
|||
|
|
+ String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);
|
|||
|
|
if (brokerController.getBrokerConfig().isEnablePopLog()) {
|
|||
|
|
POP_LOGGER.info("reviveQueueId={},find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
|
|||
|
|
}
|
|||
|
|
@@ -395,7 +395,7 @@ public class PopReviveService extends ServiceThread {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) {
|
|||
|
|
- String raw = new String(messageExt.getBody(), DataConverter.charset);
|
|||
|
|
+ String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);
|
|||
|
|
if (brokerController.getBrokerConfig().isEnablePopLog()) {
|
|||
|
|
POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
|
|||
|
|
}
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java
|
|||
|
|
index 6503970af..e9a81a8d6 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java
|
|||
|
|
@@ -30,7 +30,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {
|
|||
|
|
|
|||
|
|
public RocksDBSubscriptionGroupManager(BrokerController brokerController) {
|
|||
|
|
super(brokerController, false);
|
|||
|
|
- this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval());
|
|||
|
|
+ this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs());
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
@@ -53,7 +53,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {
|
|||
|
|
SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
- byte[] keyBytes = groupName.getBytes(DataConverter.charset);
|
|||
|
|
+ byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8);
|
|||
|
|
byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible);
|
|||
|
|
this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
|
|||
|
|
} catch (Exception e) {
|
|||
|
|
@@ -68,7 +68,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {
|
|||
|
|
SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.putIfAbsent(groupName, subscriptionGroupConfig);
|
|||
|
|
if (oldConfig == null) {
|
|||
|
|
try {
|
|||
|
|
- byte[] keyBytes = groupName.getBytes(DataConverter.charset);
|
|||
|
|
+ byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8);
|
|||
|
|
byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible);
|
|||
|
|
this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
|
|||
|
|
} catch (Exception e) {
|
|||
|
|
@@ -82,7 +82,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {
|
|||
|
|
protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) {
|
|||
|
|
SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.remove(groupName);
|
|||
|
|
try {
|
|||
|
|
- this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.charset));
|
|||
|
|
+ this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
} catch (Exception e) {
|
|||
|
|
log.error("kv delete sub Failed, {}", subscriptionGroupConfig.toString());
|
|||
|
|
}
|
|||
|
|
@@ -91,7 +91,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
protected void decode0(byte[] key, byte[] body) {
|
|||
|
|
- String groupName = new String(key, DataConverter.charset);
|
|||
|
|
+ String groupName = new String(key, DataConverter.CHARSET_UTF8);
|
|||
|
|
SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class);
|
|||
|
|
|
|||
|
|
this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);
|
|||
|
|
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java
|
|||
|
|
index 7da0d7c8a..fddecf2d9 100644
|
|||
|
|
--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java
|
|||
|
|
+++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java
|
|||
|
|
@@ -30,7 +30,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager {
|
|||
|
|
|
|||
|
|
public RocksDBTopicConfigManager(BrokerController brokerController) {
|
|||
|
|
super(brokerController, false);
|
|||
|
|
- this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval());
|
|||
|
|
+ this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs());
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
@@ -49,7 +49,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager {
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
protected void decode0(byte[] key, byte[] body) {
|
|||
|
|
- String topicName = new String(key, DataConverter.charset);
|
|||
|
|
+ String topicName = new String(key, DataConverter.CHARSET_UTF8);
|
|||
|
|
TopicConfig topicConfig = JSON.parseObject(body, TopicConfig.class);
|
|||
|
|
|
|||
|
|
this.topicConfigTable.put(topicName, topicConfig);
|
|||
|
|
@@ -66,7 +66,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager {
|
|||
|
|
String topicName = topicConfig.getTopicName();
|
|||
|
|
TopicConfig oldTopicConfig = this.topicConfigTable.put(topicName, topicConfig);
|
|||
|
|
try {
|
|||
|
|
- byte[] keyBytes = topicName.getBytes(DataConverter.charset);
|
|||
|
|
+ byte[] keyBytes = topicName.getBytes(DataConverter.CHARSET_UTF8);
|
|||
|
|
byte[] valueBytes = JSON.toJSONBytes(topicConfig, SerializerFeature.BrowserCompatible);
|
|||
|
|
this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
|
|||
|
|
} catch (Exception e) {
|
|||
|
|
@@ -79,7 +79,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager {
|
|||
|
|
protected TopicConfig removeTopicConfig(String topicName) {
|
|||
|
|
TopicConfig topicConfig = this.topicConfigTable.remove(topicName);
|
|||
|
|
try {
|
|||
|
|
- this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.charset));
|
|||
|
|
+ this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
} catch (Exception e) {
|
|||
|
|
log.error("kv remove topic Failed, {}", topicConfig.toString());
|
|||
|
|
}
|
|||
|
|
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java
|
|||
|
|
index 1c3a0cd45..78b76264f 100644
|
|||
|
|
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java
|
|||
|
|
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java
|
|||
|
|
@@ -234,7 +234,7 @@ public class PopReviveServiceTest {
|
|||
|
|
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
|
|||
|
|
|
|||
|
|
msgInner.setTopic(REVIVE_TOPIC);
|
|||
|
|
- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset));
|
|||
|
|
+ msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
msgInner.setQueueId(REVIVE_QUEUE_ID);
|
|||
|
|
msgInner.setTags(PopAckConstants.CK_TAG);
|
|||
|
|
msgInner.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
@@ -269,7 +269,7 @@ public class PopReviveServiceTest {
|
|||
|
|
SocketAddress host, long deliverMs, String ackUniqueId) {
|
|||
|
|
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
|
|||
|
|
msgInner.setTopic(reviveTopic);
|
|||
|
|
- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset));
|
|||
|
|
+ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
|
|||
|
|
msgInner.setQueueId(reviveQid);
|
|||
|
|
msgInner.setTags(PopAckConstants.ACK_TAG);
|
|||
|
|
msgInner.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
diff --git a/common/pom.xml b/common/pom.xml
|
|||
|
|
index 6104c3ac6..a28ed228f 100644
|
|||
|
|
--- a/common/pom.xml
|
|||
|
|
+++ b/common/pom.xml
|
|||
|
|
@@ -109,7 +109,7 @@
|
|||
|
|
<artifactId>rocketmq-logback-classic</artifactId>
|
|||
|
|
</dependency>
|
|||
|
|
<dependency>
|
|||
|
|
- <groupId>io.github.aliyunmq</groupId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
<artifactId>rocketmq-rocksdb</artifactId>
|
|||
|
|
</dependency>
|
|||
|
|
</dependencies>
|
|||
|
|
diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
|
|||
|
|
index 1233a5422..407ef2842 100644
|
|||
|
|
--- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java
|
|||
|
|
+++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
|
|||
|
|
@@ -492,6 +492,7 @@ public class MixAll {
|
|||
|
|
public static int compareLong(long x, long y) {
|
|||
|
|
return Long.compare(x, y);
|
|||
|
|
}
|
|||
|
|
+
|
|||
|
|
public static boolean isLmq(String lmqMetaData) {
|
|||
|
|
return lmqMetaData != null && lmqMetaData.startsWith(LMQ_PREFIX);
|
|||
|
|
}
|
|||
|
|
diff --git a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java
|
|||
|
|
index 73ef21880..9148d5a18 100644
|
|||
|
|
--- a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java
|
|||
|
|
+++ b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java
|
|||
|
|
@@ -19,5 +19,6 @@ package org.apache.rocketmq.common.attribute;
|
|||
|
|
|
|||
|
|
public enum CQType {
|
|||
|
|
SimpleCQ,
|
|||
|
|
- BatchCQ
|
|||
|
|
+ BatchCQ,
|
|||
|
|
+ RocksDBCQ
|
|||
|
|
}
|
|||
|
|
diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
|
|||
|
|
index 6f19a9815..20319abba 100644
|
|||
|
|
--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
|
|||
|
|
+++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
|
|||
|
|
@@ -17,7 +17,6 @@
|
|||
|
|
package org.apache.rocketmq.common.config;
|
|||
|
|
|
|||
|
|
import java.nio.ByteBuffer;
|
|||
|
|
-import java.nio.charset.Charset;
|
|||
|
|
import java.util.ArrayList;
|
|||
|
|
import java.util.List;
|
|||
|
|
import java.util.Map;
|
|||
|
|
@@ -27,11 +26,11 @@ import java.util.concurrent.Semaphore;
|
|||
|
|
import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
import java.util.concurrent.TimeUnit;
|
|||
|
|
|
|||
|
|
-import com.google.common.collect.Lists;
|
|||
|
|
import com.google.common.collect.Maps;
|
|||
|
|
|
|||
|
|
import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+import org.apache.rocketmq.common.utils.DataConverter;
|
|||
|
|
import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
@@ -47,7 +46,6 @@ import org.rocksdb.Priority;
|
|||
|
|
import org.rocksdb.ReadOptions;
|
|||
|
|
import org.rocksdb.RocksDB;
|
|||
|
|
import org.rocksdb.RocksDBException;
|
|||
|
|
-import org.rocksdb.RocksIterator;
|
|||
|
|
import org.rocksdb.Statistics;
|
|||
|
|
import org.rocksdb.Status;
|
|||
|
|
import org.rocksdb.WriteBatch;
|
|||
|
|
@@ -58,7 +56,6 @@ import static org.rocksdb.RocksDB.NOT_FOUND;
|
|||
|
|
public abstract class AbstractRocksDBStorage {
|
|||
|
|
protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
|
|||
|
|
|
|||
|
|
- private static final Charset CHARSET_UTF8 = Charset.forName("UTF-8");
|
|||
|
|
private static final String SPACE = " | ";
|
|||
|
|
|
|||
|
|
protected String dbPath;
|
|||
|
|
@@ -223,10 +220,6 @@ public abstract class AbstractRocksDBStorage {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- protected WrappedRocksIterator newIterator(ColumnFamilyHandle cfHandle, ReadOptions readOptions) {
|
|||
|
|
- return new WrappedRocksIterator(this.db.newIterator(cfHandle, readOptions));
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions,
|
|||
|
|
final byte[] startKey, final byte[] endKey) throws RocksDBException {
|
|||
|
|
if (!hold()) {
|
|||
|
|
@@ -243,46 +236,6 @@ public abstract class AbstractRocksDBStorage {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- protected void manualCompactionDefaultCfMaxLevel(final CompactionOptions compactionOptions) throws Exception {
|
|||
|
|
- final ColumnFamilyHandle defaultCFHandle = this.defaultCFHandle;
|
|||
|
|
- final byte[] defaultCFName = defaultCFHandle.getName();
|
|||
|
|
- List<LiveFileMetaData> fileMetaDataList = this.db.getLiveFilesMetaData();
|
|||
|
|
- if (fileMetaDataList == null || fileMetaDataList.isEmpty()) {
|
|||
|
|
- return;
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- List<LiveFileMetaData> defaultLiveFileDataList = Lists.newArrayList();
|
|||
|
|
- List<String> inputFileNames = Lists.newArrayList();
|
|||
|
|
- int maxLevel = 0;
|
|||
|
|
- for (LiveFileMetaData fileMetaData : fileMetaDataList) {
|
|||
|
|
- if (compareTo(fileMetaData.columnFamilyName(), defaultCFName) != 0) {
|
|||
|
|
- continue;
|
|||
|
|
- }
|
|||
|
|
- defaultLiveFileDataList.add(fileMetaData);
|
|||
|
|
- if (fileMetaData.level() > maxLevel) {
|
|||
|
|
- maxLevel = fileMetaData.level();
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
- if (maxLevel == 0) {
|
|||
|
|
- LOGGER.info("manualCompactionDefaultCfFiles skip level 0.");
|
|||
|
|
- return;
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- for (LiveFileMetaData fileMetaData : defaultLiveFileDataList) {
|
|||
|
|
- if (fileMetaData.level() != maxLevel || fileMetaData.beingCompacted()) {
|
|||
|
|
- continue;
|
|||
|
|
- }
|
|||
|
|
- inputFileNames.add(fileMetaData.path() + fileMetaData.fileName());
|
|||
|
|
- }
|
|||
|
|
- if (!inputFileNames.isEmpty()) {
|
|||
|
|
- List<String> outputLists = this.db.compactFiles(compactionOptions, defaultCFHandle,
|
|||
|
|
- inputFileNames, maxLevel, -1, null);
|
|||
|
|
- LOGGER.info("manualCompactionDefaultCfFiles OK. src: {}, dst: {}", inputFileNames, outputLists);
|
|||
|
|
- } else {
|
|||
|
|
- LOGGER.info("manualCompactionDefaultCfFiles Empty.");
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
protected void manualCompactionDefaultCfRange(CompactRangeOptions compactRangeOptions) {
|
|||
|
|
if (!hold()) {
|
|||
|
|
return;
|
|||
|
|
@@ -494,50 +447,6 @@ public abstract class AbstractRocksDBStorage {
|
|||
|
|
this.db.flushWal(true);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- protected class WrappedRocksIterator {
|
|||
|
|
- private final RocksIterator iterator;
|
|||
|
|
-
|
|||
|
|
- public WrappedRocksIterator(final RocksIterator iterator) {
|
|||
|
|
- this.iterator = iterator;
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public byte[] key() {
|
|||
|
|
- return iterator.key();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public byte[] value() {
|
|||
|
|
- return iterator.value();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public void next() {
|
|||
|
|
- iterator.next();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public void prev() {
|
|||
|
|
- iterator.prev();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public void seek(byte[] target) {
|
|||
|
|
- iterator.seek(target);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public void seekForPrev(byte[] target) {
|
|||
|
|
- iterator.seekForPrev(target);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public void seekToFirst() {
|
|||
|
|
- iterator.seekToFirst();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public boolean isValid() {
|
|||
|
|
- return iterator.isValid();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public void close() {
|
|||
|
|
- iterator.close();
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
private String getStatusError(RocksDBException e) {
|
|||
|
|
if (e == null || e.getStatus() == null) {
|
|||
|
|
return "null";
|
|||
|
|
@@ -574,7 +483,7 @@ public abstract class AbstractRocksDBStorage {
|
|||
|
|
sb = new StringBuilder(256);
|
|||
|
|
map.put(metaData.level(), sb);
|
|||
|
|
}
|
|||
|
|
- sb.append(new String(metaData.columnFamilyName(), CHARSET_UTF8)).append(SPACE).
|
|||
|
|
+ sb.append(new String(metaData.columnFamilyName(), DataConverter.CHARSET_UTF8)).append(SPACE).
|
|||
|
|
append(metaData.fileName()).append(SPACE).
|
|||
|
|
append("s: ").append(metaData.size()).append(SPACE).
|
|||
|
|
append("a: ").append(metaData.numEntries()).append(SPACE).
|
|||
|
|
@@ -595,21 +504,4 @@ public abstract class AbstractRocksDBStorage {
|
|||
|
|
} catch (Exception ignored) {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
-
|
|||
|
|
- public int compareTo(byte[] v1, byte[] v2) {
|
|||
|
|
- int len1 = v1.length;
|
|||
|
|
- int len2 = v2.length;
|
|||
|
|
- int lim = Math.min(len1, len2);
|
|||
|
|
-
|
|||
|
|
- int k = 0;
|
|||
|
|
- while (k < lim) {
|
|||
|
|
- byte c1 = v1[k];
|
|||
|
|
- byte c2 = v2[k];
|
|||
|
|
- if (c1 != c2) {
|
|||
|
|
- return c1 - c2;
|
|||
|
|
- }
|
|||
|
|
- k++;
|
|||
|
|
- }
|
|||
|
|
- return len1 - len2;
|
|||
|
|
- }
|
|||
|
|
}
|
|||
|
|
\ No newline at end of file
|
|||
|
|
diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java
|
|||
|
|
index 463bd8fed..b40f8046e 100644
|
|||
|
|
--- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java
|
|||
|
|
+++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java
|
|||
|
|
@@ -203,7 +203,7 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage {
|
|||
|
|
setUseDirectReads(true);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private static String getDBLogDir() {
|
|||
|
|
+ public static String getDBLogDir() {
|
|||
|
|
String rootPath = System.getProperty("user.home");
|
|||
|
|
if (StringUtils.isEmpty(rootPath)) {
|
|||
|
|
return "";
|
|||
|
|
diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java
|
|||
|
|
index 0c72ebb7b..91599653c 100644
|
|||
|
|
--- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java
|
|||
|
|
+++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java
|
|||
|
|
@@ -70,4 +70,17 @@ public class MessageExtBrokerInner extends MessageExt {
|
|||
|
|
public void setVersion(MessageVersion version) {
|
|||
|
|
this.version = version;
|
|||
|
|
}
|
|||
|
|
+
|
|||
|
|
+ public void removeWaitStorePropertyString() {
|
|||
|
|
+ if (this.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) {
|
|||
|
|
+ // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message.
|
|||
|
|
+ // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it.
|
|||
|
|
+ String waitStoreMsgOKValue = this.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK);
|
|||
|
|
+ this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties()));
|
|||
|
|
+ // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later
|
|||
|
|
+ this.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue);
|
|||
|
|
+ } else {
|
|||
|
|
+ this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties()));
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
diff --git a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java
|
|||
|
|
index 61265c05d..c19592a44 100644
|
|||
|
|
--- a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java
|
|||
|
|
+++ b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java
|
|||
|
|
@@ -31,6 +31,7 @@ public class TopicValidator {
|
|||
|
|
public static final String RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC = "TRANS_CHECK_MAX_TIME_TOPIC";
|
|||
|
|
public static final String RMQ_SYS_SELF_TEST_TOPIC = "SELF_TEST_TOPIC";
|
|||
|
|
public static final String RMQ_SYS_OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT";
|
|||
|
|
+ public static final String RMQ_SYS_ROCKSDB_OFFSET_TOPIC = "CHECKPOINT_TOPIC";
|
|||
|
|
|
|||
|
|
public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_";
|
|||
|
|
public static final String SYNC_BROKER_MEMBER_GROUP_PREFIX = SYSTEM_TOPIC_PREFIX + "SYNC_BROKER_MEMBER_";
|
|||
|
|
@@ -55,6 +56,7 @@ public class TopicValidator {
|
|||
|
|
SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC);
|
|||
|
|
SYSTEM_TOPIC_SET.add(RMQ_SYS_SELF_TEST_TOPIC);
|
|||
|
|
SYSTEM_TOPIC_SET.add(RMQ_SYS_OFFSET_MOVED_EVENT);
|
|||
|
|
+ SYSTEM_TOPIC_SET.add(RMQ_SYS_ROCKSDB_OFFSET_TOPIC);
|
|||
|
|
|
|||
|
|
NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_SCHEDULE_TOPIC);
|
|||
|
|
NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_TRANS_HALF_TOPIC);
|
|||
|
|
diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java
|
|||
|
|
index 8b50de12b..cc96770b2 100644
|
|||
|
|
--- a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java
|
|||
|
|
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java
|
|||
|
|
@@ -20,7 +20,7 @@ import java.nio.ByteBuffer;
|
|||
|
|
import java.nio.charset.Charset;
|
|||
|
|
|
|||
|
|
public class DataConverter {
|
|||
|
|
- public static Charset charset = Charset.forName("UTF-8");
|
|||
|
|
+ public static final Charset CHARSET_UTF8 = Charset.forName("UTF-8");
|
|||
|
|
|
|||
|
|
public static byte[] Long2Byte(Long v) {
|
|||
|
|
ByteBuffer tmp = ByteBuffer.allocate(8);
|
|||
|
|
diff --git a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java
|
|||
|
|
index aac295030..8662328ea 100644
|
|||
|
|
--- a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java
|
|||
|
|
+++ b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java
|
|||
|
|
@@ -71,7 +71,7 @@ public class Producer {
|
|||
|
|
TAG /* Tag */,
|
|||
|
|
("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
|
|||
|
|
);
|
|||
|
|
-
|
|||
|
|
+ msg.setDelayTimeLevel(1);
|
|||
|
|
/*
|
|||
|
|
* Call send message to deliver message to one of brokers.
|
|||
|
|
*/
|
|||
|
|
diff --git a/metrics b/metrics
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..146e90fae
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/metrics
|
|||
|
|
@@ -0,0 +1,241 @@
|
|||
|
|
+# TYPE target info
|
|||
|
|
+# HELP target Target metadata
|
|||
|
|
+target_info{} 1
|
|||
|
|
+# TYPE otel_scope_info info
|
|||
|
|
+# HELP otel_scope_info Scope metadata
|
|||
|
|
+otel_scope_info{otel_scope_name="broker-meter"} 1
|
|||
|
|
+# TYPE rocketmq_storage_message_reserve_time gauge
|
|||
|
|
+# HELP rocketmq_storage_message_reserve_time default view for gauge.
|
|||
|
|
+rocketmq_storage_message_reserve_time_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1.9491132515E10 1696905287361
|
|||
|
|
+# TYPE rocketmq_timer_enqueue_latency gauge
|
|||
|
|
+# HELP rocketmq_timer_enqueue_latency default view for gauge.
|
|||
|
|
+rocketmq_timer_enqueue_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_timer_enqueue_lag gauge
|
|||
|
|
+# HELP rocketmq_timer_enqueue_lag default view for gauge.
|
|||
|
|
+rocketmq_timer_enqueue_lag{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_timing_messages gauge
|
|||
|
|
+# HELP rocketmq_timing_messages default view for gauge.
|
|||
|
|
+rocketmq_timing_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local",topic="BenchmarkTest"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_storage_flush_behind_bytes gauge
|
|||
|
|
+# HELP rocketmq_storage_flush_behind_bytes default view for gauge.
|
|||
|
|
+rocketmq_storage_flush_behind_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1230.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_processor_watermark gauge
|
|||
|
|
+# HELP rocketmq_processor_watermark default view for gauge.
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="reply"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="transaction"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="send"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="async_put"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="lite_pull"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="pull"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="heartbeat"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="consumer_manager"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="admin"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="client_manager"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="query_message"} 0.0 1696905287361
|
|||
|
|
+rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="ack"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_pop_buffer_scan_time_consume histogram
|
|||
|
|
+# HELP rocketmq_pop_buffer_scan_time_consume Time consuming of pop buffer scan
|
|||
|
|
+rocketmq_pop_buffer_scan_time_consume_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 5267005.0 1696905287361
|
|||
|
|
+rocketmq_pop_buffer_scan_time_consume_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 39651.0 1696905287361
|
|||
|
|
+rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="1.0"} 5266935.0 1696905287361
|
|||
|
|
+rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="10.0"} 5266998.0 1696905287361
|
|||
|
|
+rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="100.0"} 5267001.0 1696905287361
|
|||
|
|
+rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="1000.0"} 5267005.0 1696905287361
|
|||
|
|
+rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="2000.0"} 5267005.0 1696905287361
|
|||
|
|
+rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="3000.0"} 5267005.0 1696905287361
|
|||
|
|
+rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="+Inf"} 5267005.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_pop_revive_lag gauge
|
|||
|
|
+# HELP rocketmq_pop_revive_lag default view for gauge.
|
|||
|
|
+rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="1"} 1.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="2"} 1.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="3"} 1.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="4"} 1.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="5"} 1.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="6"} 1.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="7"} 1.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="0"} 1.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_pop_revive_latency gauge
|
|||
|
|
+# HELP rocketmq_pop_revive_latency default view for gauge.
|
|||
|
|
+rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="1"} 0.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="2"} 0.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="3"} 0.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="4"} 0.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="5"} 0.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="6"} 0.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="7"} 0.0 1696905287361
|
|||
|
|
+rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="0"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_storage_size gauge
|
|||
|
|
+# HELP rocketmq_storage_size default view for gauge.
|
|||
|
|
+rocketmq_storage_size_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1.5905103872E10 1696905287361
|
|||
|
|
+# TYPE rocketmq_pop_offset_buffer_size gauge
|
|||
|
|
+# HELP rocketmq_pop_offset_buffer_size default view for gauge.
|
|||
|
|
+rocketmq_pop_offset_buffer_size{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_timer_dequeue_latency gauge
|
|||
|
|
+# HELP rocketmq_timer_dequeue_latency default view for gauge.
|
|||
|
|
+rocketmq_timer_dequeue_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_broker_permission gauge
|
|||
|
|
+# HELP rocketmq_broker_permission default view for gauge.
|
|||
|
|
+rocketmq_broker_permission{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 6.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_consumer_lag_messages gauge
|
|||
|
|
+# HELP rocketmq_consumer_lag_messages default view for gauge.
|
|||
|
|
+rocketmq_consumer_lag_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_throughput_out_total counter
|
|||
|
|
+# HELP rocketmq_throughput_out_total default view for counter.
|
|||
|
|
+rocketmq_throughput_out_total{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_timer_dequeue_lag gauge
|
|||
|
|
+# HELP rocketmq_timer_dequeue_lag default view for gauge.
|
|||
|
|
+rocketmq_timer_dequeue_lag{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_messages_out_total counter
|
|||
|
|
+# HELP rocketmq_messages_out_total default view for counter.
|
|||
|
|
+rocketmq_messages_out_total{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_messages_in_total counter
|
|||
|
|
+# HELP rocketmq_messages_in_total default view for counter.
|
|||
|
|
+rocketmq_messages_in_total{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_consumer_queueing_latency gauge
|
|||
|
|
+# HELP rocketmq_consumer_queueing_latency default view for gauge.
|
|||
|
|
+rocketmq_consumer_queueing_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_consumer_connections gauge
|
|||
|
|
+# HELP rocketmq_consumer_connections default view for gauge.
|
|||
|
|
+rocketmq_consumer_connections{otel_scope_name="broker-meter",cluster="DefaultCluster",consume_mode="push",consumer_group="MQ_INST_test%GID_222",is_system="false",language="java",node_id="broker-a",node_type="broker",protocol_type="remoting",version="v4_8_0"} 1.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_rpc_latency histogram
|
|||
|
|
+# HELP rocketmq_rpc_latency Rpc latency
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success"} 24.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success"} 128.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="1.0"} 17.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="3.0"} 18.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="5.0"} 19.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="7.0"} 19.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="10.0"} 20.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="100.0"} 24.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="1000.0"} 24.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="2000.0"} 24.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="3000.0"} 24.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="+Inf"} 24.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success"} 8.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="1.0"} 3.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="3.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="5.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="7.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="10.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="100.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="1000.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="2000.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="3000.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="+Inf"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success"} 54.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="1.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="3.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="5.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="7.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="10.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="100.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="1000.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="2000.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="3000.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="+Inf"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success"} 60.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success"} 108.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="1.0"} 54.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="3.0"} 54.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="5.0"} 54.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="7.0"} 54.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="10.0"} 55.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="100.0"} 60.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="1000.0"} 60.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="2000.0"} 60.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="3000.0"} 60.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="+Inf"} 60.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success"} 4.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success"} 210.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="1.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="3.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="5.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="7.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="10.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="100.0"} 4.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="1000.0"} 4.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="2000.0"} 4.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="3000.0"} 4.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="+Inf"} 4.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success"} 155.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success"} 2811756.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="1.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="3.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="5.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="7.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="10.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="100.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="1000.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="2000.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="3000.0"} 0.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="+Inf"} 155.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success"} 225.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="1.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="3.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="5.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="7.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="10.0"} 4.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="100.0"} 4.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="1000.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="2000.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="3000.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="+Inf"} 5.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway"} 565.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway"} 84.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="1.0"} 551.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="3.0"} 557.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="5.0"} 558.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="7.0"} 560.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="10.0"} 565.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="100.0"} 565.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="1000.0"} 565.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="2000.0"} 565.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="3000.0"} 565.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="+Inf"} 565.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success"} 6.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="1.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="3.0"} 1.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="5.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="7.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="10.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="100.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="1000.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="2000.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="3000.0"} 2.0 1696905287361
|
|||
|
|
+rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="+Inf"} 2.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_storage_dispatch_behind_bytes gauge
|
|||
|
|
+# HELP rocketmq_storage_dispatch_behind_bytes default view for gauge.
|
|||
|
|
+rocketmq_storage_dispatch_behind_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_consumer_ready_messages gauge
|
|||
|
|
+# HELP rocketmq_consumer_ready_messages default view for gauge.
|
|||
|
|
+rocketmq_consumer_ready_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_consumer_lag_latency gauge
|
|||
|
|
+# HELP rocketmq_consumer_lag_latency default view for gauge.
|
|||
|
|
+rocketmq_consumer_lag_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_throughput_in_total counter
|
|||
|
|
+# HELP rocketmq_throughput_in_total default view for counter.
|
|||
|
|
+rocketmq_throughput_in_total{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_pop_checkpoint_buffer_size gauge
|
|||
|
|
+# HELP rocketmq_pop_checkpoint_buffer_size default view for gauge.
|
|||
|
|
+rocketmq_pop_checkpoint_buffer_size{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_message_size histogram
|
|||
|
|
+# HELP rocketmq_message_size Incoming messages size
|
|||
|
|
+rocketmq_message_size_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361
|
|||
|
|
+rocketmq_message_size_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361
|
|||
|
|
+rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="1024.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="4096.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="524288.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="1048576.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="2097152.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="4194304.0"} 5.0 1696905287361
|
|||
|
|
+rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="+Inf"} 5.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_consumer_inflight_messages gauge
|
|||
|
|
+# HELP rocketmq_consumer_inflight_messages default view for gauge.
|
|||
|
|
+rocketmq_consumer_inflight_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361
|
|||
|
|
+# TYPE rocketmq_producer_connections gauge
|
|||
|
|
+# HELP rocketmq_producer_connections default view for gauge.
|
|||
|
|
+rocketmq_producer_connections{otel_scope_name="broker-meter",cluster="DefaultCluster",language="java",node_id="broker-a",node_type="broker",protocol_type="remoting",version="v4_8_0"} 1.0 1696905287361
|
|||
|
|
diff --git a/patch009-modify-CRLF-to-LF.patch b/patch009-modify-CRLF-to-LF.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..c9f1f1843
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch009-modify-CRLF-to-LF.patch
|
|||
|
|
@@ -0,0 +1,127 @@
|
|||
|
|
+From 81cd734ee33586544694675bd1081f2c81024d22 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: zhiliatom <zhiliatox@163.com>
|
|||
|
|
+Date: Thu, 28 Sep 2023 09:58:13 +0800
|
|||
|
|
+Subject: [PATCH] modify line separater from CRLF to LF
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/common/constant/LoggerName.java | 110 +++++++++---------
|
|||
|
|
+ 1 file changed, 55 insertions(+), 55 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
|
|||
|
|
+index cb04b00b3..5c7afab9f 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
|
|||
|
|
+@@ -1,55 +1,55 @@
|
|||
|
|
+-/*
|
|||
|
|
+- * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+- * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+- * this work for additional information regarding copyright ownership.
|
|||
|
|
+- * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+- * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+- * the License. You may obtain a copy of the License at
|
|||
|
|
+- *
|
|||
|
|
+- * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+- *
|
|||
|
|
+- * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+- * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+- * See the License for the specific language governing permissions and
|
|||
|
|
+- * limitations under the License.
|
|||
|
|
+- */
|
|||
|
|
+-package org.apache.rocketmq.common.constant;
|
|||
|
|
+-
|
|||
|
|
+-public class LoggerName {
|
|||
|
|
+- public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv";
|
|||
|
|
+- public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv";
|
|||
|
|
+- public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole";
|
|||
|
|
+- public static final String CONTROLLER_LOGGER_NAME = "RocketmqController";
|
|||
|
|
+- public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark";
|
|||
|
|
+- public static final String BROKER_LOGGER_NAME = "RocketmqBroker";
|
|||
|
|
+- public static final String BROKER_CONSOLE_NAME = "RocketmqConsole";
|
|||
|
|
+- public static final String CLIENT_LOGGER_NAME = "RocketmqClient";
|
|||
|
|
+- public static final String ROCKETMQ_TRAFFIC_NAME = "RocketmqTraffic";
|
|||
|
|
+- public static final String ROCKETMQ_REMOTING_NAME = "RocketmqRemoting";
|
|||
|
|
+- public static final String TOOLS_LOGGER_NAME = "RocketmqTools";
|
|||
|
|
+- public static final String COMMON_LOGGER_NAME = "RocketmqCommon";
|
|||
|
|
+- public static final String STORE_LOGGER_NAME = "RocketmqStore";
|
|||
|
|
+- public static final String STORE_ERROR_LOGGER_NAME = "RocketmqStoreError";
|
|||
|
|
+- public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction";
|
|||
|
|
+- public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock";
|
|||
|
|
+- public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats";
|
|||
|
|
+- public static final String DLQ_STATS_LOGGER_NAME = "RocketmqDLQStats";
|
|||
|
|
+- public static final String DLQ_LOGGER_NAME = "RocketmqDLQ";
|
|||
|
|
+- public static final String CONSUMER_STATS_LOGGER_NAME = "RocketmqConsumerStats";
|
|||
|
|
+- public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial";
|
|||
|
|
+- public static final String ACCOUNT_LOGGER_NAME = "RocketmqAccount";
|
|||
|
|
+- public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl";
|
|||
|
|
+- public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize";
|
|||
|
|
+- public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication";
|
|||
|
|
+- public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection";
|
|||
|
|
+- public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark";
|
|||
|
|
+- public static final String FILTER_LOGGER_NAME = "RocketmqFilter";
|
|||
|
|
+- public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop";
|
|||
|
|
+- public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover";
|
|||
|
|
+- public static final String STDOUT_LOGGER_NAME = "STDOUT";
|
|||
|
|
+- public static final String PROXY_LOGGER_NAME = "RocketmqProxy";
|
|||
|
|
+- public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark";
|
|||
|
|
+- public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = "RocketmqColdCtr";
|
|||
|
|
+- public static final String ROCKSDB_LOGGER_NAME = "RocketmqRocksDB";
|
|||
|
|
+-}
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++package org.apache.rocketmq.common.constant;
|
|||
|
|
++
|
|||
|
|
++public class LoggerName {
|
|||
|
|
++ public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv";
|
|||
|
|
++ public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv";
|
|||
|
|
++ public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole";
|
|||
|
|
++ public static final String CONTROLLER_LOGGER_NAME = "RocketmqController";
|
|||
|
|
++ public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark";
|
|||
|
|
++ public static final String BROKER_LOGGER_NAME = "RocketmqBroker";
|
|||
|
|
++ public static final String BROKER_CONSOLE_NAME = "RocketmqConsole";
|
|||
|
|
++ public static final String CLIENT_LOGGER_NAME = "RocketmqClient";
|
|||
|
|
++ public static final String ROCKETMQ_TRAFFIC_NAME = "RocketmqTraffic";
|
|||
|
|
++ public static final String ROCKETMQ_REMOTING_NAME = "RocketmqRemoting";
|
|||
|
|
++ public static final String TOOLS_LOGGER_NAME = "RocketmqTools";
|
|||
|
|
++ public static final String COMMON_LOGGER_NAME = "RocketmqCommon";
|
|||
|
|
++ public static final String STORE_LOGGER_NAME = "RocketmqStore";
|
|||
|
|
++ public static final String STORE_ERROR_LOGGER_NAME = "RocketmqStoreError";
|
|||
|
|
++ public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction";
|
|||
|
|
++ public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock";
|
|||
|
|
++ public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats";
|
|||
|
|
++ public static final String DLQ_STATS_LOGGER_NAME = "RocketmqDLQStats";
|
|||
|
|
++ public static final String DLQ_LOGGER_NAME = "RocketmqDLQ";
|
|||
|
|
++ public static final String CONSUMER_STATS_LOGGER_NAME = "RocketmqConsumerStats";
|
|||
|
|
++ public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial";
|
|||
|
|
++ public static final String ACCOUNT_LOGGER_NAME = "RocketmqAccount";
|
|||
|
|
++ public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl";
|
|||
|
|
++ public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize";
|
|||
|
|
++ public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication";
|
|||
|
|
++ public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection";
|
|||
|
|
++ public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark";
|
|||
|
|
++ public static final String FILTER_LOGGER_NAME = "RocketmqFilter";
|
|||
|
|
++ public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop";
|
|||
|
|
++ public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover";
|
|||
|
|
++ public static final String STDOUT_LOGGER_NAME = "STDOUT";
|
|||
|
|
++ public static final String PROXY_LOGGER_NAME = "RocketmqProxy";
|
|||
|
|
++ public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark";
|
|||
|
|
++ public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = "RocketmqColdCtr";
|
|||
|
|
++ public static final String ROCKSDB_LOGGER_NAME = "RocketmqRocksDB";
|
|||
|
|
++}
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch010-backport-add-some-fixes b/patch010-backport-add-some-fixes
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..36d5e4a1f
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch010-backport-add-some-fixes
|
|||
|
|
@@ -0,0 +1,1286 @@
|
|||
|
|
+From b2deef179dbc6a9eb1a2b6dd7b652d95cb768295 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lizhimins <707364882@qq.com>
|
|||
|
|
+Date: Thu, 10 Aug 2023 10:38:47 +0800
|
|||
|
|
+Subject: [PATCH 01/12] [ISSUE #7144] Accelerate the recovery speed of the
|
|||
|
|
+ tiered storage module (#7145)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../tieredstore/TieredDispatcher.java | 3 +
|
|||
|
|
+ .../tieredstore/TieredMessageStore.java | 2 +-
|
|||
|
|
+ .../common/TieredStoreExecutor.java | 25 ++--
|
|||
|
|
+ .../tieredstore/file/CompositeFlatFile.java | 15 +-
|
|||
|
|
+ .../file/CompositeQueueFlatFile.java | 20 ++-
|
|||
|
|
+ .../tieredstore/file/TieredCommitLog.java | 24 +++-
|
|||
|
|
+ .../tieredstore/file/TieredFlatFile.java | 42 +++---
|
|||
|
|
+ .../file/TieredFlatFileManager.java | 135 ++++++++++--------
|
|||
|
|
+ .../metadata/FileSegmentMetadata.java | 26 +++-
|
|||
|
|
+ .../tieredstore/TieredDispatcherTest.java | 15 +-
|
|||
|
|
+ .../tieredstore/TieredMessageFetcherTest.java | 2 +-
|
|||
|
|
+ .../file/CompositeQueueFlatFileTest.java | 2 +-
|
|||
|
|
+ .../file/TieredFlatFileManagerTest.java | 7 +-
|
|||
|
|
+ 13 files changed, 194 insertions(+), 124 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
|
|||
|
|
+index bb58ea7dd..1746190cd 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
|
|||
|
|
+@@ -279,6 +279,9 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch
|
|||
|
|
+ long upperBound = Math.min(dispatchOffset + maxCount, maxOffsetInQueue);
|
|||
|
|
+ ConsumeQueue consumeQueue = (ConsumeQueue) defaultStore.getConsumeQueue(topic, queueId);
|
|||
|
|
+
|
|||
|
|
++ logger.debug("DispatchFlatFile race, topic={}, queueId={}, cq range={}-{}, dispatch offset={}-{}",
|
|||
|
|
++ topic, queueId, minOffsetInQueue, maxOffsetInQueue, dispatchOffset, upperBound - 1);
|
|||
|
|
++
|
|||
|
|
+ for (; dispatchOffset < upperBound; dispatchOffset++) {
|
|||
|
|
+ // get consume queue
|
|||
|
|
+ SelectMappedBufferResult cqItem = consumeQueue.getIndexBuffer(dispatchOffset);
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+index 1f12410f2..ced1fb818 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+@@ -147,7 +147,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) {
|
|||
|
|
+
|
|||
|
|
+ if (!viaTieredStorage(topic, queueId, offset, maxMsgNums)) {
|
|||
|
|
+- logger.debug("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset);
|
|||
|
|
++ logger.trace("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset);
|
|||
|
|
+ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java
|
|||
|
|
+index 6eb3478b3..6dd0e8846 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java
|
|||
|
|
+@@ -43,18 +43,9 @@ public class TieredStoreExecutor {
|
|||
|
|
+ public static ExecutorService compactIndexFileExecutor;
|
|||
|
|
+
|
|||
|
|
+ public static void init() {
|
|||
|
|
+- dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
|
|||
|
|
+- dispatchExecutor = new ThreadPoolExecutor(
|
|||
|
|
+- Math.max(2, Runtime.getRuntime().availableProcessors()),
|
|||
|
|
+- Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
|
|||
|
|
+- 1000 * 60,
|
|||
|
|
+- TimeUnit.MILLISECONDS,
|
|||
|
|
+- dispatchThreadPoolQueue,
|
|||
|
|
+- new ThreadFactoryImpl("TieredCommonExecutor_"));
|
|||
|
|
+-
|
|||
|
|
+ commonScheduledExecutor = new ScheduledThreadPoolExecutor(
|
|||
|
|
+ Math.max(4, Runtime.getRuntime().availableProcessors()),
|
|||
|
|
+- new ThreadFactoryImpl("TieredCommonScheduledExecutor_"));
|
|||
|
|
++ new ThreadFactoryImpl("TieredCommonExecutor_"));
|
|||
|
|
+
|
|||
|
|
+ commitExecutor = new ScheduledThreadPoolExecutor(
|
|||
|
|
+ Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
|
|||
|
|
+@@ -62,7 +53,17 @@ public class TieredStoreExecutor {
|
|||
|
|
+
|
|||
|
|
+ cleanExpiredFileExecutor = new ScheduledThreadPoolExecutor(
|
|||
|
|
+ Math.max(4, Runtime.getRuntime().availableProcessors()),
|
|||
|
|
+- new ThreadFactoryImpl("TieredCleanExpiredFileExecutor_"));
|
|||
|
|
++ new ThreadFactoryImpl("TieredCleanFileExecutor_"));
|
|||
|
|
++
|
|||
|
|
++ dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
|
|||
|
|
++ dispatchExecutor = new ThreadPoolExecutor(
|
|||
|
|
++ Math.max(2, Runtime.getRuntime().availableProcessors()),
|
|||
|
|
++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
|
|||
|
|
++ 1000 * 60,
|
|||
|
|
++ TimeUnit.MILLISECONDS,
|
|||
|
|
++ dispatchThreadPoolQueue,
|
|||
|
|
++ new ThreadFactoryImpl("TieredDispatchExecutor_"),
|
|||
|
|
++ new ThreadPoolExecutor.DiscardOldestPolicy());
|
|||
|
|
+
|
|||
|
|
+ fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
|
|||
|
|
+ fetchDataExecutor = new ThreadPoolExecutor(
|
|||
|
|
+@@ -71,7 +72,7 @@ public class TieredStoreExecutor {
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+ TimeUnit.MILLISECONDS,
|
|||
|
|
+ fetchDataThreadPoolQueue,
|
|||
|
|
+- new ThreadFactoryImpl("TieredFetchDataExecutor_"));
|
|||
|
|
++ new ThreadFactoryImpl("TieredFetchExecutor_"));
|
|||
|
|
+
|
|||
|
|
+ compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
|
|||
|
|
+ compactIndexFileExecutor = new ThreadPoolExecutor(
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java
|
|||
|
|
+index df4baf33f..5ad3a6ff3 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java
|
|||
|
|
+@@ -76,20 +76,15 @@ public class CompositeFlatFile implements CompositeAccess {
|
|||
|
|
+ this.storeConfig = fileQueueFactory.getStoreConfig();
|
|||
|
|
+ this.readAheadFactor = this.storeConfig.getReadAheadMinFactor();
|
|||
|
|
+ this.metadataStore = TieredStoreUtil.getMetadataStore(this.storeConfig);
|
|||
|
|
+- this.dispatchOffset = new AtomicLong();
|
|||
|
|
+ this.compositeFlatFileLock = new ReentrantLock();
|
|||
|
|
+ this.inFlightRequestMap = new ConcurrentHashMap<>();
|
|||
|
|
+ this.commitLog = new TieredCommitLog(fileQueueFactory, filePath);
|
|||
|
|
+ this.consumeQueue = new TieredConsumeQueue(fileQueueFactory, filePath);
|
|||
|
|
++ this.dispatchOffset = new AtomicLong(
|
|||
|
|
++ this.consumeQueue.isInitialized() ? this.getConsumeQueueCommitOffset() : -1L);
|
|||
|
|
+ this.groupOffsetCache = this.initOffsetCache();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- protected void recoverMetadata() {
|
|||
|
|
+- if (!consumeQueue.isInitialized() && this.dispatchOffset.get() != -1) {
|
|||
|
|
+- consumeQueue.setBaseOffset(this.dispatchOffset.get() * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+ private Cache<String, Long> initOffsetCache() {
|
|||
|
|
+ return Caffeine.newBuilder()
|
|||
|
|
+ .expireAfterWrite(2, TimeUnit.MINUTES)
|
|||
|
|
+@@ -310,10 +305,12 @@ public class CompositeFlatFile implements CompositeAccess {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void initOffset(long offset) {
|
|||
|
|
+- if (!consumeQueue.isInitialized()) {
|
|||
|
|
++ if (consumeQueue.isInitialized()) {
|
|||
|
|
++ dispatchOffset.set(this.getConsumeQueueCommitOffset());
|
|||
|
|
++ } else {
|
|||
|
|
+ consumeQueue.setBaseOffset(offset * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
|
|||
|
|
++ dispatchOffset.set(offset);
|
|||
|
|
+ }
|
|||
|
|
+- dispatchOffset.set(offset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java
|
|||
|
|
+index f6c0afed0..0a797f465 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java
|
|||
|
|
+@@ -36,8 +36,7 @@ public class CompositeQueueFlatFile extends CompositeFlatFile {
|
|||
|
|
+ public CompositeQueueFlatFile(TieredFileAllocator fileQueueFactory, MessageQueue messageQueue) {
|
|||
|
|
+ super(fileQueueFactory, TieredStoreUtil.toPath(messageQueue));
|
|||
|
|
+ this.messageQueue = messageQueue;
|
|||
|
|
+- this.recoverTopicMetadata();
|
|||
|
|
+- super.recoverMetadata();
|
|||
|
|
++ this.recoverQueueMetadata();
|
|||
|
|
+ this.indexFile = TieredFlatFileManager.getIndexFile(storeConfig);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -46,11 +45,12 @@ public class CompositeQueueFlatFile extends CompositeFlatFile {
|
|||
|
|
+ if (!consumeQueue.isInitialized()) {
|
|||
|
|
+ queueMetadata.setMinOffset(offset);
|
|||
|
|
+ queueMetadata.setMaxOffset(offset);
|
|||
|
|
++ metadataStore.updateQueue(queueMetadata);
|
|||
|
|
+ }
|
|||
|
|
+ super.initOffset(offset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void recoverTopicMetadata() {
|
|||
|
|
++ public void recoverQueueMetadata() {
|
|||
|
|
+ TopicMetadata topicMetadata = this.metadataStore.getTopic(messageQueue.getTopic());
|
|||
|
|
+ if (topicMetadata == null) {
|
|||
|
|
+ topicMetadata = this.metadataStore.addTopic(messageQueue.getTopic(), -1L);
|
|||
|
|
+@@ -64,18 +64,16 @@ public class CompositeQueueFlatFile extends CompositeFlatFile {
|
|||
|
|
+ if (queueMetadata.getMaxOffset() < queueMetadata.getMinOffset()) {
|
|||
|
|
+ queueMetadata.setMaxOffset(queueMetadata.getMinOffset());
|
|||
|
|
+ }
|
|||
|
|
+- this.dispatchOffset.set(queueMetadata.getMaxOffset());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void persistMetadata() {
|
|||
|
|
++ public void flushMetadata() {
|
|||
|
|
+ try {
|
|||
|
|
+- if (consumeQueue.getCommitOffset() < queueMetadata.getMinOffset()) {
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- queueMetadata.setMaxOffset(consumeQueue.getCommitOffset() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
|
|||
|
|
++ queueMetadata.setMinOffset(super.getConsumeQueueMinOffset());
|
|||
|
|
++ queueMetadata.setMaxOffset(super.getConsumeQueueMaxOffset());
|
|||
|
|
+ metadataStore.updateQueue(queueMetadata);
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- LOGGER.error("CompositeFlatFile#flushMetadata: update queue metadata failed: topic: {}, queue: {}", messageQueue.getTopic(), messageQueue.getQueueId(), e);
|
|||
|
|
++ LOGGER.error("CompositeFlatFile#flushMetadata error, topic: {}, queue: {}",
|
|||
|
|
++ messageQueue.getTopic(), messageQueue.getQueueId(), e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -114,7 +112,7 @@ public class CompositeQueueFlatFile extends CompositeFlatFile {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void shutdown() {
|
|||
|
|
+ super.shutdown();
|
|||
|
|
+- metadataStore.updateQueue(queueMetadata);
|
|||
|
|
++ this.flushMetadata();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java
|
|||
|
|
+index 80e1bce50..0e5f79132 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java
|
|||
|
|
+@@ -50,7 +50,7 @@ public class TieredCommitLog {
|
|||
|
|
+ this.storeConfig = fileQueueFactory.getStoreConfig();
|
|||
|
|
+ this.flatFile = fileQueueFactory.createFlatFileForCommitLog(filePath);
|
|||
|
|
+ this.minConsumeQueueOffset = new AtomicLong(NOT_EXIST_MIN_OFFSET);
|
|||
|
|
+- this.correctMinOffset();
|
|||
|
|
++ this.correctMinOffsetAsync();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @VisibleForTesting
|
|||
|
|
+@@ -91,17 +91,26 @@ public class TieredCommitLog {
|
|||
|
|
+ return flatFile.getFileToWrite().getMaxTimestamp();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public synchronized long correctMinOffset() {
|
|||
|
|
++ public long correctMinOffset() {
|
|||
|
|
++ try {
|
|||
|
|
++ return correctMinOffsetAsync().get();
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ log.error("Correct min offset failed in clean expired file", e);
|
|||
|
|
++ }
|
|||
|
|
++ return NOT_EXIST_MIN_OFFSET;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public synchronized CompletableFuture<Long> correctMinOffsetAsync() {
|
|||
|
|
+ if (flatFile.getFileSegmentCount() == 0) {
|
|||
|
|
+ this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET);
|
|||
|
|
+- return NOT_EXIST_MIN_OFFSET;
|
|||
|
|
++ return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ // queue offset field length is 8
|
|||
|
|
+ int length = MessageBufferUtil.QUEUE_OFFSET_POSITION + 8;
|
|||
|
|
+ if (flatFile.getCommitOffset() - flatFile.getMinOffset() < length) {
|
|||
|
|
+ this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET);
|
|||
|
|
+- return NOT_EXIST_MIN_OFFSET;
|
|||
|
|
++ return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ try {
|
|||
|
|
+@@ -109,7 +118,8 @@ public class TieredCommitLog {
|
|||
|
|
+ .thenApply(buffer -> {
|
|||
|
|
+ long offset = MessageBufferUtil.getQueueOffset(buffer);
|
|||
|
|
+ minConsumeQueueOffset.set(offset);
|
|||
|
|
+- log.info("Correct commitlog min cq offset success, filePath={}, min cq offset={}, range={}-{}",
|
|||
|
|
++ log.debug("Correct commitlog min cq offset success, " +
|
|||
|
|
++ "filePath={}, min cq offset={}, commitlog range={}-{}",
|
|||
|
|
+ flatFile.getFilePath(), offset, flatFile.getMinOffset(), flatFile.getCommitOffset());
|
|||
|
|
+ return offset;
|
|||
|
|
+ })
|
|||
|
|
+@@ -117,11 +127,11 @@ public class TieredCommitLog {
|
|||
|
|
+ log.warn("Correct commitlog min cq offset error, filePath={}, range={}-{}",
|
|||
|
|
+ flatFile.getFilePath(), flatFile.getMinOffset(), flatFile.getCommitOffset(), throwable);
|
|||
|
|
+ return minConsumeQueueOffset.get();
|
|||
|
|
+- }).get();
|
|||
|
|
++ });
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ log.error("Correct commitlog min cq offset error, filePath={}", flatFile.getFilePath(), e);
|
|||
|
|
+ }
|
|||
|
|
+- return minConsumeQueueOffset.get();
|
|||
|
|
++ return CompletableFuture.completedFuture(minConsumeQueueOffset.get());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public AppendResult append(ByteBuffer byteBuf) {
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
|
|||
|
|
+index 75ce8d89f..426c4e09d 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
|
|||
|
|
+@@ -16,6 +16,7 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.tieredstore.file;
|
|||
|
|
+
|
|||
|
|
++import com.alibaba.fastjson.JSON;
|
|||
|
|
+ import com.google.common.annotations.VisibleForTesting;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+@@ -24,6 +25,7 @@ import java.util.Comparator;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+ import java.util.LinkedList;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
++import java.util.Objects;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.CopyOnWriteArrayList;
|
|||
|
|
+@@ -178,32 +180,26 @@ public class TieredFlatFile {
|
|||
|
|
+ private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fileSegment) {
|
|||
|
|
+
|
|||
|
|
+ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment(
|
|||
|
|
+- fileSegment.getPath(), fileSegment.getFileType(), fileSegment.getBaseOffset());
|
|||
|
|
+-
|
|||
|
|
+- if (metadata != null) {
|
|||
|
|
+- return metadata;
|
|||
|
|
+- }
|
|||
|
|
++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset());
|
|||
|
|
+
|
|||
|
|
+ // Note: file segment path may not the same as file base path, use base path here.
|
|||
|
|
+- metadata = new FileSegmentMetadata(
|
|||
|
|
+- this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType());
|
|||
|
|
+-
|
|||
|
|
+- if (fileSegment.isClosed()) {
|
|||
|
|
+- metadata.setStatus(FileSegmentMetadata.STATUS_DELETED);
|
|||
|
|
++ if (metadata == null) {
|
|||
|
|
++ metadata = new FileSegmentMetadata(
|
|||
|
|
++ this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType());
|
|||
|
|
++ metadata.setCreateTimestamp(fileSegment.getMinTimestamp());
|
|||
|
|
++ metadata.setBeginTimestamp(fileSegment.getMinTimestamp());
|
|||
|
|
++ metadata.setEndTimestamp(fileSegment.getMaxTimestamp());
|
|||
|
|
++ if (fileSegment.isClosed()) {
|
|||
|
|
++ metadata.setStatus(FileSegmentMetadata.STATUS_DELETED);
|
|||
|
|
++ }
|
|||
|
|
++ this.tieredMetadataStore.updateFileSegment(metadata);
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
+- metadata.setBeginTimestamp(fileSegment.getMinTimestamp());
|
|||
|
|
+- metadata.setEndTimestamp(fileSegment.getMaxTimestamp());
|
|||
|
|
+-
|
|||
|
|
+- // Submit to persist
|
|||
|
|
+- this.tieredMetadataStore.updateFileSegment(metadata);
|
|||
|
|
+ return metadata;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full
|
|||
|
|
+ */
|
|||
|
|
+- @VisibleForTesting
|
|||
|
|
+ public void updateFileSegment(TieredFileSegment fileSegment) {
|
|||
|
|
+ FileSegmentMetadata segmentMetadata = getOrCreateFileSegmentMetadata(fileSegment);
|
|||
|
|
+
|
|||
|
|
+@@ -219,9 +215,16 @@ public class TieredFlatFile {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ segmentMetadata.setSize(fileSegment.getCommitPosition());
|
|||
|
|
+- segmentMetadata.setBeginTimestamp(fileSegment.getMinTimestamp());
|
|||
|
|
+ segmentMetadata.setEndTimestamp(fileSegment.getMaxTimestamp());
|
|||
|
|
+- this.tieredMetadataStore.updateFileSegment(segmentMetadata);
|
|||
|
|
++
|
|||
|
|
++ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment(
|
|||
|
|
++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset());
|
|||
|
|
++
|
|||
|
|
++ if (!Objects.equals(metadata, segmentMetadata)) {
|
|||
|
|
++ this.tieredMetadataStore.updateFileSegment(segmentMetadata);
|
|||
|
|
++ logger.debug("TieredFlatFile#UpdateSegmentMetadata, filePath: {}, content: {}",
|
|||
|
|
++ segmentMetadata.getPath(), JSON.toJSONString(segmentMetadata));
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void checkAndFixFileSize() {
|
|||
|
|
+@@ -257,6 +260,7 @@ public class TieredFlatFile {
|
|||
|
|
+ logger.warn("TieredFlatFile#checkAndFixFileSize: fix last file {} size: origin: {}, actual: {}",
|
|||
|
|
+ lastFile.getPath(), lastFile.getCommitOffset() - lastFile.getBaseOffset(), lastFileSize);
|
|||
|
|
+ lastFile.initPosition(lastFileSize);
|
|||
|
|
++ this.updateFileSegment(lastFile);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
|
|||
|
|
+index aeca44b8c..e9ae4a5a5 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
|
|||
|
|
+@@ -16,16 +16,19 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.tieredstore.file;
|
|||
|
|
+
|
|||
|
|
++import com.google.common.base.Stopwatch;
|
|||
|
|
+ import com.google.common.collect.ImmutableList;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Random;
|
|||
|
|
++import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+-import java.util.concurrent.Future;
|
|||
|
|
++import java.util.concurrent.Semaphore;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicLong;
|
|||
|
|
+ import javax.annotation.Nullable;
|
|||
|
|
++import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+@@ -36,6 +39,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+
|
|||
|
|
+ public class TieredFlatFileManager {
|
|||
|
|
+
|
|||
|
|
++ private static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
|
|||
|
|
+ private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
+ private static volatile TieredFlatFileManager instance;
|
|||
|
|
+@@ -44,7 +48,7 @@ public class TieredFlatFileManager {
|
|||
|
|
+ private final TieredMetadataStore metadataStore;
|
|||
|
|
+ private final TieredMessageStoreConfig storeConfig;
|
|||
|
|
+ private final TieredFileAllocator tieredFileAllocator;
|
|||
|
|
+- private final ConcurrentMap<MessageQueue, CompositeQueueFlatFile> queueFlatFileMap;
|
|||
|
|
++ private final ConcurrentMap<MessageQueue, CompositeQueueFlatFile> flatFileConcurrentMap;
|
|||
|
|
+
|
|||
|
|
+ public TieredFlatFileManager(TieredMessageStoreConfig storeConfig)
|
|||
|
|
+ throws ClassNotFoundException, NoSuchMethodException {
|
|||
|
|
+@@ -52,23 +56,20 @@ public class TieredFlatFileManager {
|
|||
|
|
+ this.storeConfig = storeConfig;
|
|||
|
|
+ this.metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
|
|||
|
|
+ this.tieredFileAllocator = new TieredFileAllocator(storeConfig);
|
|||
|
|
+- this.queueFlatFileMap = new ConcurrentHashMap<>();
|
|||
|
|
++ this.flatFileConcurrentMap = new ConcurrentHashMap<>();
|
|||
|
|
+ this.doScheduleTask();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static TieredFlatFileManager getInstance(TieredMessageStoreConfig storeConfig) {
|
|||
|
|
+- if (storeConfig == null) {
|
|||
|
|
++ if (storeConfig == null || instance != null) {
|
|||
|
|
+ return instance;
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
+- if (instance == null) {
|
|||
|
|
+- synchronized (TieredFlatFileManager.class) {
|
|||
|
|
+- if (instance == null) {
|
|||
|
|
+- try {
|
|||
|
|
+- instance = new TieredFlatFileManager(storeConfig);
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- logger.error("TieredFlatFileManager#getInstance: create flat file manager failed", e);
|
|||
|
|
+- }
|
|||
|
|
++ synchronized (TieredFlatFileManager.class) {
|
|||
|
|
++ if (instance == null) {
|
|||
|
|
++ try {
|
|||
|
|
++ instance = new TieredFlatFileManager(storeConfig);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ logger.error("Construct FlatFileManager instance error", e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+@@ -88,7 +89,7 @@ public class TieredFlatFileManager {
|
|||
|
|
+ TieredStoreUtil.RMQ_SYS_TIERED_STORE_INDEX_TOPIC, storeConfig.getBrokerName(), 0));
|
|||
|
|
+ indexFile = new TieredIndexFile(new TieredFileAllocator(storeConfig), filePath);
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- logger.error("TieredFlatFileManager#getIndexFile: create index file failed", e);
|
|||
|
|
++ logger.error("Construct FlatFileManager indexFile error", e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+@@ -105,7 +106,7 @@ public class TieredFlatFileManager {
|
|||
|
|
+ flatFile.commitCommitLog();
|
|||
|
|
+ } catch (Throwable e) {
|
|||
|
|
+ MessageQueue mq = flatFile.getMessageQueue();
|
|||
|
|
+- logger.error("commit commitLog periodically failed: topic: {}, queue: {}",
|
|||
|
|
++ logger.error("Commit commitLog periodically failed: topic: {}, queue: {}",
|
|||
|
|
+ mq.getTopic(), mq.getQueueId(), e);
|
|||
|
|
+ }
|
|||
|
|
+ }, delay, TimeUnit.MILLISECONDS);
|
|||
|
|
+@@ -114,7 +115,7 @@ public class TieredFlatFileManager {
|
|||
|
|
+ flatFile.commitConsumeQueue();
|
|||
|
|
+ } catch (Throwable e) {
|
|||
|
|
+ MessageQueue mq = flatFile.getMessageQueue();
|
|||
|
|
+- logger.error("commit consumeQueue periodically failed: topic: {}, queue: {}",
|
|||
|
|
++ logger.error("Commit consumeQueue periodically failed: topic: {}, queue: {}",
|
|||
|
|
+ mq.getTopic(), mq.getQueueId(), e);
|
|||
|
|
+ }
|
|||
|
|
+ }, delay, TimeUnit.MILLISECONDS);
|
|||
|
|
+@@ -125,7 +126,7 @@ public class TieredFlatFileManager {
|
|||
|
|
+ indexFile.commit(true);
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Throwable e) {
|
|||
|
|
+- logger.error("commit indexFile periodically failed", e);
|
|||
|
|
++ logger.error("Commit indexFile periodically failed", e);
|
|||
|
|
+ }
|
|||
|
|
+ }, 0, TimeUnit.MILLISECONDS);
|
|||
|
|
+ }
|
|||
|
|
+@@ -160,7 +161,7 @@ public class TieredFlatFileManager {
|
|||
|
|
+ try {
|
|||
|
|
+ doCommit();
|
|||
|
|
+ } catch (Throwable e) {
|
|||
|
|
+- logger.error("commit flat file periodically failed: ", e);
|
|||
|
|
++ logger.error("Commit flat file periodically failed: ", e);
|
|||
|
|
+ }
|
|||
|
|
+ }, 60, 60, TimeUnit.SECONDS);
|
|||
|
|
+
|
|||
|
|
+@@ -168,49 +169,73 @@ public class TieredFlatFileManager {
|
|||
|
|
+ try {
|
|||
|
|
+ doCleanExpiredFile();
|
|||
|
|
+ } catch (Throwable e) {
|
|||
|
|
+- logger.error("clean expired flat file failed: ", e);
|
|||
|
|
++ logger.error("Clean expired flat file failed: ", e);
|
|||
|
|
+ }
|
|||
|
|
+ }, 30, 30, TimeUnit.SECONDS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public boolean load() {
|
|||
|
|
++ Stopwatch stopwatch = Stopwatch.createStarted();
|
|||
|
|
+ try {
|
|||
|
|
+- AtomicLong topicSequenceNumber = new AtomicLong();
|
|||
|
|
+- List<Future<?>> futureList = new ArrayList<>();
|
|||
|
|
+- queueFlatFileMap.clear();
|
|||
|
|
+- metadataStore.iterateTopic(topicMetadata -> {
|
|||
|
|
++ flatFileConcurrentMap.clear();
|
|||
|
|
++ this.recoverSequenceNumber();
|
|||
|
|
++ this.recoverTieredFlatFile();
|
|||
|
|
++ logger.info("Message store recover end, total cost={}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ long costTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
|
|||
|
|
++ logger.info("Message store recover error, total cost={}ms", costTime);
|
|||
|
|
++ BROKER_LOG.error("Message store recover error, total cost={}ms", costTime, e);
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++ return true;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void recoverSequenceNumber() {
|
|||
|
|
++ AtomicLong topicSequenceNumber = new AtomicLong();
|
|||
|
|
++ metadataStore.iterateTopic(topicMetadata -> {
|
|||
|
|
++ if (topicMetadata != null && topicMetadata.getTopicId() > 0) {
|
|||
|
|
+ topicSequenceNumber.set(Math.max(topicSequenceNumber.get(), topicMetadata.getTopicId()));
|
|||
|
|
+- Future<?> future = TieredStoreExecutor.dispatchExecutor.submit(() -> {
|
|||
|
|
+- if (topicMetadata.getStatus() != 0) {
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
++ }
|
|||
|
|
++ });
|
|||
|
|
++ metadataStore.setTopicSequenceNumber(topicSequenceNumber.incrementAndGet());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void recoverTieredFlatFile() {
|
|||
|
|
++ Semaphore semaphore = new Semaphore((int) (TieredStoreExecutor.QUEUE_CAPACITY * 0.75));
|
|||
|
|
++ List<CompletableFuture<Void>> futures = new ArrayList<>();
|
|||
|
|
++ metadataStore.iterateTopic(topicMetadata -> {
|
|||
|
|
++ try {
|
|||
|
|
++ semaphore.acquire();
|
|||
|
|
++ CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
|
|||
|
|
+ try {
|
|||
|
|
+- metadataStore.iterateQueue(topicMetadata.getTopic(),
|
|||
|
|
+- queueMetadata -> getOrCreateFlatFileIfAbsent(
|
|||
|
|
+- new MessageQueue(topicMetadata.getTopic(),
|
|||
|
|
+- storeConfig.getBrokerName(),
|
|||
|
|
+- queueMetadata.getQueue().getQueueId())));
|
|||
|
|
++ Stopwatch subWatch = Stopwatch.createStarted();
|
|||
|
|
++ if (topicMetadata.getStatus() != 0) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ AtomicLong queueCount = new AtomicLong();
|
|||
|
|
++ metadataStore.iterateQueue(topicMetadata.getTopic(), queueMetadata -> {
|
|||
|
|
++ this.getOrCreateFlatFileIfAbsent(new MessageQueue(topicMetadata.getTopic(),
|
|||
|
|
++ storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId()));
|
|||
|
|
++ queueCount.incrementAndGet();
|
|||
|
|
++ });
|
|||
|
|
++ logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms",
|
|||
|
|
++ topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS));
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- logger.error("load mq composite flat file from metadata failed", e);
|
|||
|
|
++ logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e);
|
|||
|
|
++ } finally {
|
|||
|
|
++ semaphore.release();
|
|||
|
|
+ }
|
|||
|
|
+- });
|
|||
|
|
+- futureList.add(future);
|
|||
|
|
+- });
|
|||
|
|
+-
|
|||
|
|
+- // Wait for load all metadata done
|
|||
|
|
+- for (Future<?> future : futureList) {
|
|||
|
|
+- future.get();
|
|||
|
|
++ }, TieredStoreExecutor.commitExecutor);
|
|||
|
|
++ futures.add(future);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ throw new RuntimeException(e);
|
|||
|
|
+ }
|
|||
|
|
+- metadataStore.setTopicSequenceNumber(topicSequenceNumber.incrementAndGet());
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- logger.error("load mq composite flat file from metadata failed", e);
|
|||
|
|
+- return false;
|
|||
|
|
+- }
|
|||
|
|
+- return true;
|
|||
|
|
++ });
|
|||
|
|
++ CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void cleanup() {
|
|||
|
|
+- queueFlatFileMap.clear();
|
|||
|
|
++ flatFileConcurrentMap.clear();
|
|||
|
|
+ cleanStaticReference();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -221,27 +246,25 @@ public class TieredFlatFileManager {
|
|||
|
|
+
|
|||
|
|
+ @Nullable
|
|||
|
|
+ public CompositeQueueFlatFile getOrCreateFlatFileIfAbsent(MessageQueue messageQueue) {
|
|||
|
|
+- return queueFlatFileMap.computeIfAbsent(messageQueue, mq -> {
|
|||
|
|
++ return flatFileConcurrentMap.computeIfAbsent(messageQueue, mq -> {
|
|||
|
|
+ try {
|
|||
|
|
+- logger.debug("TieredFlatFileManager#getOrCreateFlatFileIfAbsent: " +
|
|||
|
|
+- "try to create new flat file: topic: {}, queueId: {}",
|
|||
|
|
++ logger.debug("Create new TopicFlatFile, topic: {}, queueId: {}",
|
|||
|
|
+ messageQueue.getTopic(), messageQueue.getQueueId());
|
|||
|
|
+ return new CompositeQueueFlatFile(tieredFileAllocator, mq);
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- logger.error("TieredFlatFileManager#getOrCreateFlatFileIfAbsent: " +
|
|||
|
|
+- "create new flat file: topic: {}, queueId: {}",
|
|||
|
|
++ logger.debug("Create new TopicFlatFile failed, topic: {}, queueId: {}",
|
|||
|
|
+ messageQueue.getTopic(), messageQueue.getQueueId(), e);
|
|||
|
|
+- return null;
|
|||
|
|
+ }
|
|||
|
|
++ return null;
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CompositeQueueFlatFile getFlatFile(MessageQueue messageQueue) {
|
|||
|
|
+- return queueFlatFileMap.get(messageQueue);
|
|||
|
|
++ return flatFileConcurrentMap.get(messageQueue);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public ImmutableList<CompositeQueueFlatFile> deepCopyFlatFileToList() {
|
|||
|
|
+- return ImmutableList.copyOf(queueFlatFileMap.values());
|
|||
|
|
++ return ImmutableList.copyOf(flatFileConcurrentMap.values());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void shutdown() {
|
|||
|
|
+@@ -270,7 +293,7 @@ public class TieredFlatFileManager {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ // delete memory reference
|
|||
|
|
+- CompositeQueueFlatFile flatFile = queueFlatFileMap.remove(mq);
|
|||
|
|
++ CompositeQueueFlatFile flatFile = flatFileConcurrentMap.remove(mq);
|
|||
|
|
+ if (flatFile != null) {
|
|||
|
|
+ MessageQueue messageQueue = flatFile.getMessageQueue();
|
|||
|
|
+ logger.info("TieredFlatFileManager#destroyCompositeFile: " +
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java
|
|||
|
|
+index 1b232fc75..2f0fd71de 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java
|
|||
|
|
+@@ -16,6 +16,8 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.tieredstore.metadata;
|
|||
|
|
+
|
|||
|
|
++import java.util.Objects;
|
|||
|
|
++
|
|||
|
|
+ public class FileSegmentMetadata {
|
|||
|
|
+
|
|||
|
|
+ public static final int STATUS_NEW = 0;
|
|||
|
|
+@@ -43,7 +45,6 @@ public class FileSegmentMetadata {
|
|||
|
|
+ this.baseOffset = baseOffset;
|
|||
|
|
+ this.type = type;
|
|||
|
|
+ this.status = STATUS_NEW;
|
|||
|
|
+- this.createTimestamp = System.currentTimeMillis();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void markSealed() {
|
|||
|
|
+@@ -122,4 +123,27 @@ public class FileSegmentMetadata {
|
|||
|
|
+ public void setSealTimestamp(long sealTimestamp) {
|
|||
|
|
+ this.sealTimestamp = sealTimestamp;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean equals(Object o) {
|
|||
|
|
++ if (this == o)
|
|||
|
|
++ return true;
|
|||
|
|
++ if (o == null || getClass() != o.getClass())
|
|||
|
|
++ return false;
|
|||
|
|
++ FileSegmentMetadata metadata = (FileSegmentMetadata) o;
|
|||
|
|
++ return size == metadata.size
|
|||
|
|
++ && baseOffset == metadata.baseOffset
|
|||
|
|
++ && status == metadata.status
|
|||
|
|
++ && path.equals(metadata.path)
|
|||
|
|
++ && type == metadata.type
|
|||
|
|
++ && createTimestamp == metadata.createTimestamp
|
|||
|
|
++ && beginTimestamp == metadata.beginTimestamp
|
|||
|
|
++ && endTimestamp == metadata.endTimestamp
|
|||
|
|
++ && sealTimestamp == metadata.sealTimestamp;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public int hashCode() {
|
|||
|
|
++ return Objects.hash(type, path, baseOffset, status, size, createTimestamp, beginTimestamp, endTimestamp, sealTimestamp);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java
|
|||
|
|
+index e6adef1d1..5791dc9a4 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java
|
|||
|
|
+@@ -116,19 +116,20 @@ public class TieredDispatcherTest {
|
|||
|
|
+ buffer3.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 9);
|
|||
|
|
+ flatFile.appendCommitLog(buffer3);
|
|||
|
|
+ flatFile.commitCommitLog();
|
|||
|
|
+- Assert.assertEquals(10, flatFile.getDispatchOffset());
|
|||
|
|
++ Assert.assertEquals(9 + 1, flatFile.getDispatchOffset());
|
|||
|
|
++ Assert.assertEquals(9, flatFile.getCommitLogDispatchCommitOffset());
|
|||
|
|
+
|
|||
|
|
+ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer1);
|
|||
|
|
+ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer2);
|
|||
|
|
+ dispatcher.buildConsumeQueueAndIndexFile();
|
|||
|
|
+ Assert.assertEquals(7, flatFile.getConsumeQueueMaxOffset());
|
|||
|
|
+- Assert.assertEquals(7, flatFile.getDispatchOffset());
|
|||
|
|
+
|
|||
|
|
+ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 7, 7, 0, 0, buffer1);
|
|||
|
|
+ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer2);
|
|||
|
|
+ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer3);
|
|||
|
|
+ dispatcher.buildConsumeQueueAndIndexFile();
|
|||
|
|
+- Assert.assertEquals(10, flatFile.getConsumeQueueMaxOffset());
|
|||
|
|
++ Assert.assertEquals(6, flatFile.getConsumeQueueMinOffset());
|
|||
|
|
++ Assert.assertEquals(9 + 1, flatFile.getConsumeQueueMaxOffset());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+@@ -142,6 +143,7 @@ public class TieredDispatcherTest {
|
|||
|
|
+ Mockito.when(defaultStore.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(0L);
|
|||
|
|
+ Mockito.when(defaultStore.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(9L);
|
|||
|
|
+
|
|||
|
|
++ // mock cq item, position = 7
|
|||
|
|
+ ByteBuffer cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE);
|
|||
|
|
+ cqItem.putLong(7);
|
|||
|
|
+ cqItem.putInt(MessageBufferUtilTest.MSG_LEN);
|
|||
|
|
+@@ -150,13 +152,13 @@ public class TieredDispatcherTest {
|
|||
|
|
+ SelectMappedBufferResult mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null);
|
|||
|
|
+ Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(6)).thenReturn(mockResult);
|
|||
|
|
+
|
|||
|
|
++ // mock cq item, position = 8
|
|||
|
|
+ cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE);
|
|||
|
|
+ cqItem.putLong(8);
|
|||
|
|
+ cqItem.putInt(MessageBufferUtilTest.MSG_LEN);
|
|||
|
|
+ cqItem.putLong(1);
|
|||
|
|
+ cqItem.flip();
|
|||
|
|
+ mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null);
|
|||
|
|
+-
|
|||
|
|
+ Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(7)).thenReturn(mockResult);
|
|||
|
|
+
|
|||
|
|
+ mockResult = new SelectMappedBufferResult(0, MessageBufferUtilTest.buildMockedMessageBuffer(), MessageBufferUtilTest.MSG_LEN, null);
|
|||
|
|
+@@ -167,7 +169,10 @@ public class TieredDispatcherTest {
|
|||
|
|
+ mockResult = new SelectMappedBufferResult(0, msg, MessageBufferUtilTest.MSG_LEN, null);
|
|||
|
|
+ Mockito.when(defaultStore.selectOneMessageByOffset(8, MessageBufferUtilTest.MSG_LEN)).thenReturn(mockResult);
|
|||
|
|
+
|
|||
|
|
+- dispatcher.dispatchFlatFile(flatFileManager.getOrCreateFlatFileIfAbsent(mq));
|
|||
|
|
++ CompositeQueueFlatFile flatFile = flatFileManager.getOrCreateFlatFileIfAbsent(mq);
|
|||
|
|
++ Assert.assertNotNull(flatFile);
|
|||
|
|
++ flatFile.initOffset(7);
|
|||
|
|
++ dispatcher.dispatchFlatFile(flatFile);
|
|||
|
|
+ Assert.assertEquals(8, flatFileManager.getFlatFile(mq).getDispatchOffset());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java
|
|||
|
|
+index d75b2f916..774c6cf64 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java
|
|||
|
|
+@@ -23,6 +23,7 @@ import java.util.Objects;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.commons.lang3.SystemUtils;
|
|||
|
|
+ import org.apache.commons.lang3.tuple.Triple;
|
|||
|
|
++import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
+ import org.apache.rocketmq.store.GetMessageResult;
|
|||
|
|
+@@ -40,7 +41,6 @@ import org.apache.rocketmq.tieredstore.file.TieredIndexFile;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+-import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+ import org.awaitility.Awaitility;
|
|||
|
|
+ import org.junit.After;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java
|
|||
|
|
+index 27efe111e..2e028ada3 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java
|
|||
|
|
+@@ -119,7 +119,7 @@ public class CompositeQueueFlatFileTest {
|
|||
|
|
+ Assert.assertEquals(AppendResult.SUCCESS, result);
|
|||
|
|
+
|
|||
|
|
+ file.commit(true);
|
|||
|
|
+- file.persistMetadata();
|
|||
|
|
++ file.flushMetadata();
|
|||
|
|
+
|
|||
|
|
+ QueueMetadata queueMetadata = metadataStore.getQueue(mq);
|
|||
|
|
+ Assert.assertEquals(53, queueMetadata.getMaxOffset());
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java
|
|||
|
|
+index b7528c5e4..20fe4dd70 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java
|
|||
|
|
+@@ -72,10 +72,15 @@ public class TieredFlatFileManagerTest {
|
|||
|
|
+
|
|||
|
|
+ CompositeFlatFile flatFile = flatFileManager.getFlatFile(mq);
|
|||
|
|
+ Assert.assertNotNull(flatFile);
|
|||
|
|
+- Assert.assertEquals(100, flatFile.getDispatchOffset());
|
|||
|
|
++ Assert.assertEquals(-1L, flatFile.getDispatchOffset());
|
|||
|
|
++ flatFile.initOffset(100L);
|
|||
|
|
++ Assert.assertEquals(100L, flatFile.getDispatchOffset());
|
|||
|
|
++ flatFile.initOffset(200L);
|
|||
|
|
++ Assert.assertEquals(100L, flatFile.getDispatchOffset());
|
|||
|
|
+
|
|||
|
|
+ CompositeFlatFile flatFile1 = flatFileManager.getFlatFile(mq1);
|
|||
|
|
+ Assert.assertNotNull(flatFile1);
|
|||
|
|
++ flatFile1.initOffset(200L);
|
|||
|
|
+ Assert.assertEquals(200, flatFile1.getDispatchOffset());
|
|||
|
|
+
|
|||
|
|
+ flatFileManager.destroyCompositeFile(mq);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 99b39a35f29e491862296d56b7938a995d153974 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: ShuangxiDing <dingshuangxi888@gmail.com>
|
|||
|
|
+Date: Thu, 10 Aug 2023 11:28:39 +0800
|
|||
|
|
+Subject: [PATCH 02/12] [ISSUE #7115] Fix grpc response message NPE (#7116)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java
|
|||
|
|
+index 0b3c85ea6..efa879a9c 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java
|
|||
|
|
+@@ -92,7 +92,7 @@ public class ResponseBuilder {
|
|||
|
|
+ public Status buildStatus(Code code, String message) {
|
|||
|
|
+ return Status.newBuilder()
|
|||
|
|
+ .setCode(code)
|
|||
|
|
+- .setMessage(message)
|
|||
|
|
++ .setMessage(message != null ? message : code.name())
|
|||
|
|
+ .build();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From c0ba453f38183266cf9a69be754e620311e1923b Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: caigy <csgytsai@163.com>
|
|||
|
|
+Date: Thu, 10 Aug 2023 14:08:17 +0800
|
|||
|
|
+Subject: [PATCH 03/12] [ISSUE #7129] Fix resource collisions in acl tests
|
|||
|
|
+ (#7130)
|
|||
|
|
+
|
|||
|
|
+* run acl tests sequencially to avoid collision
|
|||
|
|
+
|
|||
|
|
+* disable reuseForks for acl like broker
|
|||
|
|
+
|
|||
|
|
+* Revert "[ISSUE #7135] Temporarily ignoring plainAccessValidator test (#7135)"
|
|||
|
|
+
|
|||
|
|
+This reverts commit 6bc2c8474a0ce1e2833c82dffea7b1d8f718fcd7.
|
|||
|
|
+---
|
|||
|
|
+ acl/pom.xml | 13 +++++++++++++
|
|||
|
|
+ .../acl/plain/PlainAccessControlFlowTest.java | 5 -----
|
|||
|
|
+ .../acl/plain/PlainAccessValidatorTest.java | 3 ---
|
|||
|
|
+ .../acl/plain/PlainPermissionManagerTest.java | 3 ---
|
|||
|
|
+ 4 files changed, 13 insertions(+), 11 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/acl/pom.xml b/acl/pom.xml
|
|||
|
|
+index 67bfcb8d2..989c0cf77 100644
|
|||
|
|
+--- a/acl/pom.xml
|
|||
|
|
++++ b/acl/pom.xml
|
|||
|
|
+@@ -74,4 +74,17 @@
|
|||
|
|
+ <scope>test</scope>
|
|||
|
|
+ </dependency>
|
|||
|
|
+ </dependencies>
|
|||
|
|
++
|
|||
|
|
++ <build>
|
|||
|
|
++ <plugins>
|
|||
|
|
++ <plugin>
|
|||
|
|
++ <artifactId>maven-surefire-plugin</artifactId>
|
|||
|
|
++ <version>${maven-surefire-plugin.version}</version>
|
|||
|
|
++ <configuration>
|
|||
|
|
++ <forkCount>1</forkCount>
|
|||
|
|
++ <reuseForks>false</reuseForks>
|
|||
|
|
++ </configuration>
|
|||
|
|
++ </plugin>
|
|||
|
|
++ </plugins>
|
|||
|
|
++ </build>
|
|||
|
|
+ </project>
|
|||
|
|
+diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java
|
|||
|
|
+index e7fd0932f..519345714 100644
|
|||
|
|
+--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java
|
|||
|
|
++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java
|
|||
|
|
+@@ -31,7 +31,6 @@ import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+-import org.junit.Ignore;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
+ import java.io.File;
|
|||
|
|
+@@ -44,7 +43,6 @@ import java.util.Collections;
|
|||
|
|
+ import java.util.LinkedList;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+
|
|||
|
|
+-
|
|||
|
|
+ /**
|
|||
|
|
+ * <p> In this class, we'll test the following scenarios, each containing several consecutive operations on ACL,
|
|||
|
|
+ * <p> like updating and deleting ACL, changing config files and checking validations.
|
|||
|
|
+@@ -52,9 +50,6 @@ import java.util.List;
|
|||
|
|
+ * <p> Case 2: Only conf/acl/plain_acl.yml exists;
|
|||
|
|
+ * <p> Case 3: Both conf/plain_acl.yml and conf/acl/plain_acl.yml exists.
|
|||
|
|
+ */
|
|||
|
|
+-
|
|||
|
|
+-// Ignore this test case as it is currently unable to pass on ubuntu workflow
|
|||
|
|
+-@Ignore
|
|||
|
|
+ public class PlainAccessControlFlowTest {
|
|||
|
|
+ public static final String DEFAULT_TOPIC = "topic-acl";
|
|||
|
|
+
|
|||
|
|
+diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java
|
|||
|
|
+index a3a925758..ef0cffbdc 100644
|
|||
|
|
+--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java
|
|||
|
|
++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java
|
|||
|
|
+@@ -56,11 +56,8 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
|
|||
|
|
+ import org.junit.After;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+-import org.junit.Ignore;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
+-// Ignore this test case as it is currently unable to pass on ubuntu workflow
|
|||
|
|
+-@Ignore
|
|||
|
|
+ public class PlainAccessValidatorTest {
|
|||
|
|
+
|
|||
|
|
+ private PlainAccessValidator plainAccessValidator;
|
|||
|
|
+diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java
|
|||
|
|
+index aa7539f3a..941d8c779 100644
|
|||
|
|
+--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java
|
|||
|
|
++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java
|
|||
|
|
+@@ -29,7 +29,6 @@ import org.assertj.core.api.Assertions;
|
|||
|
|
+ import org.assertj.core.util.Lists;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+-import org.junit.Ignore;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
+ import java.io.File;
|
|||
|
|
+@@ -42,8 +41,6 @@ import java.util.List;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+
|
|||
|
|
+-// Ignore this test case as it is currently unable to pass on ubuntu workflow
|
|||
|
|
+-@Ignore
|
|||
|
|
+ public class PlainPermissionManagerTest {
|
|||
|
|
+
|
|||
|
|
+ PlainPermissionManager plainPermissionManager;
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 8741ff8c9b3bdbfc97976285affa7ea35c81243c Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: ShuangxiDing <dingshuangxi888@gmail.com>
|
|||
|
|
+Date: Thu, 10 Aug 2023 17:41:15 +0800
|
|||
|
|
+Subject: [PATCH 04/12] [ISSUE #7153] Add switch for MIXED message type (#7154)
|
|||
|
|
+
|
|||
|
|
+Add a switch for MIXED message type when creating a Topic in the Broker.
|
|||
|
|
+---
|
|||
|
|
+ .../broker/processor/AdminBrokerProcessor.java | 8 ++++++++
|
|||
|
|
+ .../java/org/apache/rocketmq/common/BrokerConfig.java | 10 ++++++++++
|
|||
|
|
+ 2 files changed, 18 insertions(+)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
+index a6ce03dc2..bbddcec2d 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
+@@ -59,6 +59,7 @@ import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.UnlockCallback;
|
|||
|
|
+ import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
+ import org.apache.rocketmq.common.attribute.AttributeParser;
|
|||
|
|
++import org.apache.rocketmq.common.attribute.TopicMessageType;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.ConsumeInitMode;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.FIleReadaheadMode;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+@@ -439,6 +440,13 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
+ String attributesModification = requestHeader.getAttributes();
|
|||
|
|
+ topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification));
|
|||
|
|
+
|
|||
|
|
++ if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED
|
|||
|
|
++ && !brokerController.getBrokerConfig().isEnableMixedMessageType()) {
|
|||
|
|
++ response.setCode(ResponseCode.SYSTEM_ERROR);
|
|||
|
|
++ response.setRemark("MIXED message type is not supported.");
|
|||
|
|
++ return response;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ try {
|
|||
|
|
+ this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);
|
|||
|
|
+ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
|
|||
|
|
+index a815636b1..99a5db5ad 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
|
|||
|
|
+@@ -393,6 +393,8 @@ public class BrokerConfig extends BrokerIdentity {
|
|||
|
|
+ */
|
|||
|
|
+ private boolean enableSingleTopicRegister = false;
|
|||
|
|
+
|
|||
|
|
++ private boolean enableMixedMessageType = false;
|
|||
|
|
++
|
|||
|
|
+ public long getMaxPopPollingSize() {
|
|||
|
|
+ return maxPopPollingSize;
|
|||
|
|
+ }
|
|||
|
|
+@@ -1712,4 +1714,12 @@ public class BrokerConfig extends BrokerIdentity {
|
|||
|
|
+ public void setEnableSingleTopicRegister(boolean enableSingleTopicRegister) {
|
|||
|
|
+ this.enableSingleTopicRegister = enableSingleTopicRegister;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isEnableMixedMessageType() {
|
|||
|
|
++ return enableMixedMessageType;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setEnableMixedMessageType(boolean enableMixedMessageType) {
|
|||
|
|
++ this.enableMixedMessageType = enableMixedMessageType;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From f534501855f8edbcb58f5b856973bf1027b5cf3a Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Steven <shirenchuang@users.noreply.github.com>
|
|||
|
|
+Date: Fri, 11 Aug 2023 10:25:48 +0800
|
|||
|
|
+Subject: [PATCH 05/12] [Feature 7155] add errlog when cmd err (#7157)
|
|||
|
|
+MIME-Version: 1.0
|
|||
|
|
+Content-Type: text/plain; charset=UTF-8
|
|||
|
|
+Content-Transfer-Encoding: 8bit
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: 十真 <shirenchuang.src@cainiao.com>
|
|||
|
|
+---
|
|||
|
|
+ .../src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java | 1 +
|
|||
|
|
+ 1 file changed, 1 insertion(+)
|
|||
|
|
+
|
|||
|
|
+diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java
|
|||
|
|
+index b00bad3c5..5a8a7cd54 100644
|
|||
|
|
+--- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java
|
|||
|
|
++++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java
|
|||
|
|
+@@ -52,6 +52,7 @@ public class ServerUtil {
|
|||
|
|
+ System.exit(0);
|
|||
|
|
+ }
|
|||
|
|
+ } catch (ParseException e) {
|
|||
|
|
++ System.err.println(e.getMessage());
|
|||
|
|
+ hf.printHelp(appName, options, true);
|
|||
|
|
+ System.exit(1);
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From db58f00c0fe0f129611d654291f2177de55dc9ff Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Zhouxiang Zhan <zhouxzhan@apache.org>
|
|||
|
|
+Date: Fri, 11 Aug 2023 19:18:30 +0800
|
|||
|
|
+Subject: [PATCH 06/12] [ISSUE #7169] Change metadataThreadPoolQueueCapacity to
|
|||
|
|
+ 100000 (#7170)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+index 4f57a7052..39caaa0d9 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+@@ -165,7 +165,7 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ private int subscriptionGroupConfigCacheExpiredInSeconds = 20;
|
|||
|
|
+ private int subscriptionGroupConfigCacheMaxNum = 20000;
|
|||
|
|
+ private int metadataThreadPoolNums = 3;
|
|||
|
|
+- private int metadataThreadPoolQueueCapacity = 1000;
|
|||
|
|
++ private int metadataThreadPoolQueueCapacity = 100000;
|
|||
|
|
+
|
|||
|
|
+ private int transactionHeartbeatThreadPoolNums = 20;
|
|||
|
|
+ private int transactionHeartbeatThreadPoolQueueCapacity = 200;
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 1f04e68a2e331ab035b791280c5a91b60fe0c85f Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yx9o <yangx_soft@163.com>
|
|||
|
|
+Date: Sat, 12 Aug 2023 21:12:22 +0800
|
|||
|
|
+Subject: [PATCH 07/12] [ISSUE #7172] Unified Chinese for Name Server (#7173)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ docs/cn/concept.md | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/docs/cn/concept.md b/docs/cn/concept.md
|
|||
|
|
+index cb2c863bd..3d67e9371 100644
|
|||
|
|
+--- a/docs/cn/concept.md
|
|||
|
|
++++ b/docs/cn/concept.md
|
|||
|
|
+@@ -17,7 +17,7 @@ RocketMQ主要由 Producer、Broker、Consumer 三部分组成，其中Producer
|
|||
|
|
+ 消息中转角色，负责存储消息、转发消息。代理服务器在RocketMQ系统中负责接收从生产者发送来的消息并存储、同时为消费者的拉取请求作准备。代理服务器也存储消息相关的元数据，包括消费者组、消费进度偏移和主题和队列消息等。
|
|||
|
|
+
|
|||
|
|
+ ## 6 名字服务（Name Server）
|
|||
|
|
+- 名称服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。
|
|||
|
|
++名字服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。
|
|||
|
|
+
|
|||
|
|
+ ## 7 拉取式消费（Pull Consumer）
|
|||
|
|
+ Consumer消费的一种类型，应用通常主动调用Consumer的拉消息方法从Broker服务器拉消息、主动权由应用控制。一旦获取了批量消息，应用就会启动消费过程。
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 25005060bbace477eeaaf4c0142cece5213efbbf Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yx9o <yangx_soft@163.com>
|
|||
|
|
+Date: Sun, 13 Aug 2023 20:52:17 +0800
|
|||
|
|
+Subject: [PATCH 08/12] [ISSUE #7176] Correct mismatched logs (#7177)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
|
|||
|
|
+index 0055a1cc8..f7a95f0a6 100644
|
|||
|
|
+--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
|
|||
|
|
++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
|
|||
|
|
+@@ -522,7 +522,7 @@ public class RouteInfoManager {
|
|||
|
|
+ this.lock.writeLock().unlock();
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- log.error("wipeWritePermOfBrokerByLock Exception", e);
|
|||
|
|
++ log.error("addWritePermOfBrokerByLock Exception", e);
|
|||
|
|
+ }
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From ac411daa27117e9115a8fc5e2d5753085f009ed9 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yx9o <yangx_soft@163.com>
|
|||
|
|
+Date: Tue, 15 Aug 2023 08:31:00 +0800
|
|||
|
|
+Subject: [PATCH 09/12] [ISSUE #7183] Correct mismatched commandDesc (#7184)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../tools/command/topic/RemappingStaticTopicSubCommand.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java
|
|||
|
|
+index 849f680d0..2a08fdb5b 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java
|
|||
|
|
+@@ -47,7 +47,7 @@ public class RemappingStaticTopicSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Update or create static topic, which has fixed number of queues";
|
|||
|
|
++ return "Remapping static topic.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 55e0cdb2af3ab75a6d892f919d60797f17a99fda Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: redlsz <szliu0927@gmail.com>
|
|||
|
|
+Date: Tue, 15 Aug 2023 19:19:45 +0800
|
|||
|
|
+Subject: [PATCH 10/12] fix: IndexOutOfBoundsException when process pop
|
|||
|
|
+ response (#7003)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../org/apache/rocketmq/client/impl/MQClientAPIImpl.java | 5 ++++-
|
|||
|
|
+ .../rocketmq/proxy/service/message/LocalMessageService.java | 5 ++++-
|
|||
|
|
+ .../rocketmq/remoting/protocol/header/ExtraInfoUtil.java | 4 ++++
|
|||
|
|
+ 3 files changed, 12 insertions(+), 2 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+index 708a6acd1..5101ffc8e 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+@@ -1174,7 +1174,10 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)));
|
|||
|
|
+ continue;
|
|||
|
|
+ }
|
|||
|
|
+- key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId());
|
|||
|
|
++ // Value of POP_CK is used to determine whether it is a pop retry,
|
|||
|
|
++ // cause topic could be rewritten by broker.
|
|||
|
|
++ key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(),
|
|||
|
|
++ messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId());
|
|||
|
|
+ if (!sortMap.containsKey(key)) {
|
|||
|
|
+ sortMap.put(key, new ArrayList<>(4));
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
|
|||
|
|
+index 115c140ff..eb2c4d9ee 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
|
|||
|
|
+@@ -249,7 +249,10 @@ public class LocalMessageService implements MessageService {
|
|||
|
|
+ // <topicMark@queueId, msg queueOffset>
|
|||
|
|
+ Map<String, List<Long>> sortMap = new HashMap<>(16);
|
|||
|
|
+ for (MessageExt messageExt : messageExtList) {
|
|||
|
|
+- String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId());
|
|||
|
|
++ // Value of POP_CK is used to determine whether it is a pop retry,
|
|||
|
|
++ // cause topic could be rewritten by broker.
|
|||
|
|
++ String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(),
|
|||
|
|
++ messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId());
|
|||
|
|
+ if (!sortMap.containsKey(key)) {
|
|||
|
|
+ sortMap.put(key, new ArrayList<>(4));
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java
|
|||
|
|
+index 9a5fa89ab..13094331e 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java
|
|||
|
|
+@@ -282,6 +282,10 @@ public class ExtraInfoUtil {
|
|||
|
|
+ return (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) ? RETRY_TOPIC : NORMAL_TOPIC) + "@" + key;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public static String getStartOffsetInfoMapKey(String topic, String popCk, long key) {
|
|||
|
|
++ return ((topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || popCk != null) ? RETRY_TOPIC : NORMAL_TOPIC) + "@" + key;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public static String getQueueOffsetKeyValueKey(long queueId, long queueOffset) {
|
|||
|
|
+ return QUEUE_OFFSET + queueId + "%" + queueOffset;
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From a9c0b43f7f6ce5acfc4f2f3069553071fa93dfee Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yx9o <yangx_soft@163.com>
|
|||
|
|
+Date: Wed, 16 Aug 2023 18:45:00 +0800
|
|||
|
|
+Subject: [PATCH 11/12] [ISSUE #7192] Correct typos (#7193)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../tools/command/consumer/ConsumerProgressSubCommand.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java
|
|||
|
|
+index f51a24673..97125b854 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java
|
|||
|
|
+@@ -54,7 +54,7 @@ public class ConsumerProgressSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Query consumers's progress, speed";
|
|||
|
|
++ return "Query consumer's progress, speed.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 5a3de926b816db5a121c1d788430072a3bc942ae Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Zhouxiang Zhan <zhouxzhan@apache.org>
|
|||
|
|
+Date: Wed, 16 Aug 2023 20:52:53 +0800
|
|||
|
|
+Subject: [PATCH 12/12] Optimize updateSubscription check exist loop (#7190)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../broker/client/ConsumerGroupInfo.java | 17 ++++++-----------
|
|||
|
|
+ 1 file changed, 6 insertions(+), 11 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java
|
|||
|
|
+index 867b9c720..1ea58c125 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java
|
|||
|
|
+@@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.client;
|
|||
|
|
+
|
|||
|
|
+ import io.netty.channel.Channel;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
++import java.util.HashSet;
|
|||
|
|
+ import java.util.Iterator;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Map.Entry;
|
|||
|
|
+@@ -172,7 +173,7 @@ public class ConsumerGroupInfo {
|
|||
|
|
+ */
|
|||
|
|
+ public boolean updateSubscription(final Set<SubscriptionData> subList) {
|
|||
|
|
+ boolean updated = false;
|
|||
|
|
+-
|
|||
|
|
++ Set<String> topicSet = new HashSet<>();
|
|||
|
|
+ for (SubscriptionData sub : subList) {
|
|||
|
|
+ SubscriptionData old = this.subscriptionTable.get(sub.getTopic());
|
|||
|
|
+ if (old == null) {
|
|||
|
|
+@@ -194,22 +195,16 @@ public class ConsumerGroupInfo {
|
|||
|
|
+
|
|||
|
|
+ this.subscriptionTable.put(sub.getTopic(), sub);
|
|||
|
|
+ }
|
|||
|
|
++ // Add all new topics to the HashSet
|
|||
|
|
++ topicSet.add(sub.getTopic());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ Iterator<Entry<String, SubscriptionData>> it = this.subscriptionTable.entrySet().iterator();
|
|||
|
|
+ while (it.hasNext()) {
|
|||
|
|
+ Entry<String, SubscriptionData> next = it.next();
|
|||
|
|
+ String oldTopic = next.getKey();
|
|||
|
|
+-
|
|||
|
|
+- boolean exist = false;
|
|||
|
|
+- for (SubscriptionData sub : subList) {
|
|||
|
|
+- if (sub.getTopic().equals(oldTopic)) {
|
|||
|
|
+- exist = true;
|
|||
|
|
+- break;
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- if (!exist) {
|
|||
|
|
++ // Check HashSet with O(1) time complexity
|
|||
|
|
++ if (!topicSet.contains(oldTopic)) {
|
|||
|
|
+ log.warn("subscription changed, group: {} remove topic {} {}",
|
|||
|
|
+ this.groupName,
|
|||
|
|
+ oldTopic,
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch011-backport-optimize-config b/patch011-backport-optimize-config
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..fc8e2eceb
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch011-backport-optimize-config
|
|||
|
|
@@ -0,0 +1,1390 @@
|
|||
|
|
+From 50d1050437ed8748f86ee50261b50a1e1f63162e Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Jixiang Jin <lollipop@apache.org>
|
|||
|
|
+Date: Wed, 16 Aug 2023 21:15:00 +0800
|
|||
|
|
+Subject: [PATCH 1/7] To config the cardinalityLimit for openTelemetry metrics
|
|||
|
|
+ exporting and fix logging config for metrics (#7196)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ WORKSPACE | 14 +++---
|
|||
|
|
+ .../broker/metrics/BrokerMetricsManager.java | 47 ++++++++++++++-----
|
|||
|
|
+ .../broker/metrics/PopMetricsManager.java | 11 +++--
|
|||
|
|
+ .../src/main/resources/rmq.broker.logback.xml | 17 ++++---
|
|||
|
|
+ .../apache/rocketmq/common/BrokerConfig.java | 9 ++++
|
|||
|
|
+ .../metrics/ControllerMetricsManager.java | 6 +--
|
|||
|
|
+ pom.xml | 4 +-
|
|||
|
|
+ .../metrics/RemotingMetricsManager.java | 10 ++--
|
|||
|
|
+ .../rocketmq/store/DefaultMessageStore.java | 24 +++++-----
|
|||
|
|
+ .../apache/rocketmq/store/MessageStore.java | 6 +--
|
|||
|
|
+ .../metrics/DefaultStoreMetricsManager.java | 4 +-
|
|||
|
|
+ .../plugin/AbstractPluginMessageStore.java | 6 +--
|
|||
|
|
+ .../tieredstore/TieredMessageStore.java | 6 +--
|
|||
|
|
+ .../metrics/TieredStoreMetricsManager.java | 23 +++++----
|
|||
|
|
+ 14 files changed, 110 insertions(+), 77 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/WORKSPACE b/WORKSPACE
|
|||
|
|
+index a8a0aafe9..3126f2d1d 100644
|
|||
|
|
+--- a/WORKSPACE
|
|||
|
|
++++ b/WORKSPACE
|
|||
|
|
+@@ -88,14 +88,14 @@ maven_install(
|
|||
|
|
+ "io.grpc:grpc-api:1.47.0",
|
|||
|
|
+ "io.grpc:grpc-testing:1.47.0",
|
|||
|
|
+ "org.springframework:spring-core:5.3.26",
|
|||
|
|
+- "io.opentelemetry:opentelemetry-exporter-otlp:1.19.0",
|
|||
|
|
+- "io.opentelemetry:opentelemetry-exporter-prometheus:1.19.0-alpha",
|
|||
|
|
+- "io.opentelemetry:opentelemetry-exporter-logging:1.19.0",
|
|||
|
|
+- "io.opentelemetry:opentelemetry-sdk:1.19.0",
|
|||
|
|
++ "io.opentelemetry:opentelemetry-exporter-otlp:1.29.0",
|
|||
|
|
++ "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha",
|
|||
|
|
++ "io.opentelemetry:opentelemetry-exporter-logging:1.29.0",
|
|||
|
|
++ "io.opentelemetry:opentelemetry-sdk:1.29.0",
|
|||
|
|
+ "com.squareup.okio:okio-jvm:3.0.0",
|
|||
|
|
+- "io.opentelemetry:opentelemetry-api:1.19.0",
|
|||
|
|
+- "io.opentelemetry:opentelemetry-sdk-metrics:1.19.0",
|
|||
|
|
+- "io.opentelemetry:opentelemetry-sdk-common:1.19.0",
|
|||
|
|
++ "io.opentelemetry:opentelemetry-api:1.29.0",
|
|||
|
|
++ "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0",
|
|||
|
|
++ "io.opentelemetry:opentelemetry-sdk-common:1.29.0",
|
|||
|
|
+ "io.github.aliyunmq:rocketmq-slf4j-api:1.0.0",
|
|||
|
|
+ "io.github.aliyunmq:rocketmq-logback-classic:1.0.0",
|
|||
|
|
+ "org.slf4j:jul-to-slf4j:2.0.6",
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
|
|||
|
|
+index f0b76107e..6af5afc14 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
|
|||
|
|
+@@ -34,8 +34,10 @@ import io.opentelemetry.sdk.metrics.InstrumentType;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.SdkMeterProvider;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil;
|
|||
|
|
+ import io.opentelemetry.sdk.resources.Resource;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.Arrays;
|
|||
|
|
+@@ -361,22 +363,45 @@ public class BrokerMetricsManager {
|
|||
|
|
+ .setType(InstrumentType.HISTOGRAM)
|
|||
|
|
+ .setName(HISTOGRAM_MESSAGE_SIZE)
|
|||
|
|
+ .build();
|
|||
|
|
+- View messageSizeView = View.builder()
|
|||
|
|
+- .setAggregation(Aggregation.explicitBucketHistogram(messageSizeBuckets))
|
|||
|
|
+- .build();
|
|||
|
|
+- providerBuilder.registerView(messageSizeSelector, messageSizeView);
|
|||
|
|
+-
|
|||
|
|
+- for (Pair<InstrumentSelector, View> selectorViewPair : RemotingMetricsManager.getMetricsView()) {
|
|||
|
|
+- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2());
|
|||
|
|
++ ViewBuilder messageSizeViewBuilder = View.builder()
|
|||
|
|
++ .setAggregation(Aggregation.explicitBucketHistogram(messageSizeBuckets));
|
|||
|
|
++ // To config the cardinalityLimit for openTelemetry metrics exporting.
|
|||
|
|
++ SdkMeterProviderUtil.setCardinalityLimit(messageSizeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
|
|||
|
|
++ providerBuilder.registerView(messageSizeSelector, messageSizeViewBuilder.build());
|
|||
|
|
++
|
|||
|
|
++ for (Pair<InstrumentSelector, ViewBuilder> selectorViewPair : RemotingMetricsManager.getMetricsView()) {
|
|||
|
|
++ ViewBuilder viewBuilder = selectorViewPair.getObject2();
|
|||
|
|
++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
|
|||
|
|
++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- for (Pair<InstrumentSelector, View> selectorViewPair : messageStore.getMetricsView()) {
|
|||
|
|
+- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2());
|
|||
|
|
++ for (Pair<InstrumentSelector, ViewBuilder> selectorViewPair : messageStore.getMetricsView()) {
|
|||
|
|
++ ViewBuilder viewBuilder = selectorViewPair.getObject2();
|
|||
|
|
++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
|
|||
|
|
++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- for (Pair<InstrumentSelector, View> selectorViewPair : PopMetricsManager.getMetricsView()) {
|
|||
|
|
+- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2());
|
|||
|
|
++ for (Pair<InstrumentSelector, ViewBuilder> selectorViewPair : PopMetricsManager.getMetricsView()) {
|
|||
|
|
++ ViewBuilder viewBuilder = selectorViewPair.getObject2();
|
|||
|
|
++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
|
|||
|
|
++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build());
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ // default view builder for all counter.
|
|||
|
|
++ InstrumentSelector defaultCounterSelector = InstrumentSelector.builder()
|
|||
|
|
++ .setType(InstrumentType.COUNTER)
|
|||
|
|
++ .build();
|
|||
|
|
++ ViewBuilder defaultCounterViewBuilder = View.builder().setDescription("default view for counter.");
|
|||
|
|
++ SdkMeterProviderUtil.setCardinalityLimit(defaultCounterViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
|
|||
|
|
++ providerBuilder.registerView(defaultCounterSelector, defaultCounterViewBuilder.build());
|
|||
|
|
++
|
|||
|
|
++ //default view builder for all observable gauge.
|
|||
|
|
++ InstrumentSelector defaultGaugeSelector = InstrumentSelector.builder()
|
|||
|
|
++ .setType(InstrumentType.OBSERVABLE_GAUGE)
|
|||
|
|
++ .build();
|
|||
|
|
++ ViewBuilder defaultGaugeViewBuilder = View.builder().setDescription("default view for gauge.");
|
|||
|
|
++ SdkMeterProviderUtil.setCardinalityLimit(defaultGaugeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
|
|||
|
|
++ providerBuilder.registerView(defaultGaugeSelector, defaultGaugeViewBuilder.build());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void initStatsMetrics() {
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
|
|||
|
|
+index 463371d7e..2de220da1 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
|
|||
|
|
+@@ -27,6 +27,7 @@ import io.opentelemetry.sdk.metrics.Aggregation;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentType;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import java.time.Duration;
|
|||
|
|
+ import java.util.Arrays;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+@@ -63,7 +64,7 @@ public class PopMetricsManager {
|
|||
|
|
+ private static LongCounter popReviveGetTotal = new NopLongCounter();
|
|||
|
|
+ private static LongCounter popReviveRetryMessageTotal = new NopLongCounter();
|
|||
|
|
+
|
|||
|
|
+- public static List<Pair<InstrumentSelector, View>> getMetricsView() {
|
|||
|
|
++ public static List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {
|
|||
|
|
+ List<Double> rpcCostTimeBuckets = Arrays.asList(
|
|||
|
|
+ (double) Duration.ofMillis(1).toMillis(),
|
|||
|
|
+ (double) Duration.ofMillis(10).toMillis(),
|
|||
|
|
+@@ -76,10 +77,10 @@ public class PopMetricsManager {
|
|||
|
|
+ .setType(InstrumentType.HISTOGRAM)
|
|||
|
|
+ .setName(HISTOGRAM_POP_BUFFER_SCAN_TIME_CONSUME)
|
|||
|
|
+ .build();
|
|||
|
|
+- View popBufferScanTimeConsumeView = View.builder()
|
|||
|
|
+- .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets))
|
|||
|
|
+- .build();
|
|||
|
|
+- return Lists.newArrayList(new Pair<>(popBufferScanTimeConsumeSelector, popBufferScanTimeConsumeView));
|
|||
|
|
++ ViewBuilder popBufferScanTimeConsumeViewBuilder = View.builder()
|
|||
|
|
++ .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets));
|
|||
|
|
++
|
|||
|
|
++ return Lists.newArrayList(new Pair<>(popBufferScanTimeConsumeSelector, popBufferScanTimeConsumeViewBuilder));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static void initMetrics(Meter meter, BrokerController brokerController,
|
|||
|
|
+diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml
|
|||
|
|
+index 7d49f6664..3c51e59d4 100644
|
|||
|
|
+--- a/broker/src/main/resources/rmq.broker.logback.xml
|
|||
|
|
++++ b/broker/src/main/resources/rmq.broker.logback.xml
|
|||
|
|
+@@ -559,27 +559,27 @@
|
|||
|
|
+ </sift>
|
|||
|
|
+ </appender>
|
|||
|
|
+
|
|||
|
|
+- <appender name="RocketmqBrokerMetricsAppender" class="ch.qos.logback.classic.sift.SiftingAppender">
|
|||
|
|
++ <appender name="RocketmqBrokerMetricsSiftingAppender_inner" class="ch.qos.logback.classic.sift.SiftingAppender">
|
|||
|
|
+ <discriminator>
|
|||
|
|
+ <key>brokerContainerLogDir</key>
|
|||
|
|
+ <defaultValue>${file.separator}</defaultValue>
|
|||
|
|
+ </discriminator>
|
|||
|
|
+ <sift>
|
|||
|
|
+- <appender name="RocketmqCommercialAppender"
|
|||
|
|
++ <appender name="RocketmqBrokerMetricsAppender"
|
|||
|
|
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
|
|||
|
|
+ <file>
|
|||
|
|
+- ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_metric.log
|
|||
|
|
++ ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_metrics.log
|
|||
|
|
+ </file>
|
|||
|
|
+ <append>true</append>
|
|||
|
|
+ <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
|
|||
|
|
+ <fileNamePattern>
|
|||
|
|
+- ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_metric.%i.log.gz
|
|||
|
|
++ ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_metrics.%i.log.gz
|
|||
|
|
+ </fileNamePattern>
|
|||
|
|
+ <minIndex>1</minIndex>
|
|||
|
|
+- <maxIndex>10</maxIndex>
|
|||
|
|
++ <maxIndex>3</maxIndex>
|
|||
|
|
+ </rollingPolicy>
|
|||
|
|
+ <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
|
|||
|
|
+- <maxFileSize>500MB</maxFileSize>
|
|||
|
|
++ <maxFileSize>512MB</maxFileSize>
|
|||
|
|
+ </triggeringPolicy>
|
|||
|
|
+ <encoder>
|
|||
|
|
+ <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
|
|||
|
|
+@@ -588,6 +588,9 @@
|
|||
|
|
+ </appender>
|
|||
|
|
+ </sift>
|
|||
|
|
+ </appender>
|
|||
|
|
++ <appender name="RocketmqBrokerMetricsSiftingAppender" class="ch.qos.logback.classic.AsyncAppender">
|
|||
|
|
++ <appender-ref ref="RocketmqBrokerMetricsSiftingAppender_inner"/>
|
|||
|
|
++ </appender>
|
|||
|
|
+
|
|||
|
|
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
|||
|
|
+ <encoder>
|
|||
|
|
+@@ -670,7 +673,7 @@
|
|||
|
|
+ </logger>
|
|||
|
|
+
|
|||
|
|
+ <logger name="io.opentelemetry.exporter.logging.LoggingMetricExporter" additivity="false" level="INFO">
|
|||
|
|
+- <appender-ref ref="RocketmqBrokerMetricsAppender"/>
|
|||
|
|
++ <appender-ref ref="RocketmqBrokerMetricsSiftingAppender"/>
|
|||
|
|
+ </logger>
|
|||
|
|
+
|
|||
|
|
+ <root level="INFO">
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
|
|||
|
|
+index 99a5db5ad..45d26b29c 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
|
|||
|
|
+@@ -350,6 +350,7 @@ public class BrokerConfig extends BrokerIdentity {
|
|||
|
|
+
|
|||
|
|
+ private MetricsExporterType metricsExporterType = MetricsExporterType.DISABLE;
|
|||
|
|
+
|
|||
|
|
++ private int metricsOtelCardinalityLimit = 50 * 1000;
|
|||
|
|
+ private String metricsGrpcExporterTarget = "";
|
|||
|
|
+ private String metricsGrpcExporterHeader = "";
|
|||
|
|
+ private long metricGrpcExporterTimeOutInMills = 3 * 1000;
|
|||
|
|
+@@ -1531,6 +1532,14 @@ public class BrokerConfig extends BrokerIdentity {
|
|||
|
|
+ this.metricsExporterType = MetricsExporterType.valueOf(metricsExporterType);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public int getMetricsOtelCardinalityLimit() {
|
|||
|
|
++ return metricsOtelCardinalityLimit;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setMetricsOtelCardinalityLimit(int metricsOtelCardinalityLimit) {
|
|||
|
|
++ this.metricsOtelCardinalityLimit = metricsOtelCardinalityLimit;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public String getMetricsGrpcExporterTarget() {
|
|||
|
|
+ return metricsGrpcExporterTarget;
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java
|
|||
|
|
+index 9b30a3b43..650740bcc 100644
|
|||
|
|
+--- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java
|
|||
|
|
++++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java
|
|||
|
|
+@@ -203,7 +203,7 @@ public class ControllerMetricsManager {
|
|||
|
|
+ 10 * s
|
|||
|
|
+ );
|
|||
|
|
+
|
|||
|
|
+- View latecyView = View.builder()
|
|||
|
|
++ View latencyView = View.builder()
|
|||
|
|
+ .setAggregation(Aggregation.explicitBucketHistogram(latencyBuckets))
|
|||
|
|
+ .build();
|
|||
|
|
+
|
|||
|
|
+@@ -217,8 +217,8 @@ public class ControllerMetricsManager {
|
|||
|
|
+ .setName(HISTOGRAM_DLEDGER_OP_LATENCY)
|
|||
|
|
+ .build();
|
|||
|
|
+
|
|||
|
|
+- providerBuilder.registerView(requestLatencySelector, latecyView);
|
|||
|
|
+- providerBuilder.registerView(dLedgerOpLatencySelector, latecyView);
|
|||
|
|
++ providerBuilder.registerView(requestLatencySelector, latencyView);
|
|||
|
|
++ providerBuilder.registerView(dLedgerOpLatencySelector, latencyView);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void initMetric(Meter meter) {
|
|||
|
|
+diff --git a/pom.xml b/pom.xml
|
|||
|
|
+index 3a08d75f2..9f0b3eb96 100644
|
|||
|
|
+--- a/pom.xml
|
|||
|
|
++++ b/pom.xml
|
|||
|
|
+@@ -133,8 +133,8 @@
|
|||
|
|
+ <caffeine.version>2.9.3</caffeine.version>
|
|||
|
|
+ <spring.version>5.3.27</spring.version>
|
|||
|
|
+ <okio-jvm.version>3.0.0</okio-jvm.version>
|
|||
|
|
+- <opentelemetry.version>1.26.0</opentelemetry.version>
|
|||
|
|
+- <opentelemetry-exporter-prometheus.version>1.26.0-alpha</opentelemetry-exporter-prometheus.version>
|
|||
|
|
++ <opentelemetry.version>1.29.0</opentelemetry.version>
|
|||
|
|
++ <opentelemetry-exporter-prometheus.version>1.29.0-alpha</opentelemetry-exporter-prometheus.version>
|
|||
|
|
+ <jul-to-slf4j.version>2.0.6</jul-to-slf4j.version>
|
|||
|
|
+ <s3.version>2.20.29</s3.version>
|
|||
|
|
+ <rocksdb.version>1.0.3</rocksdb.version>
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java b/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java
|
|||
|
|
+index 34136f94f..2e0d70856 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java
|
|||
|
|
+@@ -26,6 +26,7 @@ import io.opentelemetry.sdk.metrics.Aggregation;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentType;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import java.time.Duration;
|
|||
|
|
+ import java.util.Arrays;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+@@ -61,7 +62,7 @@ public class RemotingMetricsManager {
|
|||
|
|
+ .build();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public static List<Pair<InstrumentSelector, View>> getMetricsView() {
|
|||
|
|
++ public static List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {
|
|||
|
|
+ List<Double> rpcCostTimeBuckets = Arrays.asList(
|
|||
|
|
+ (double) Duration.ofMillis(1).toMillis(),
|
|||
|
|
+ (double) Duration.ofMillis(3).toMillis(),
|
|||
|
|
+@@ -77,10 +78,9 @@ public class RemotingMetricsManager {
|
|||
|
|
+ .setType(InstrumentType.HISTOGRAM)
|
|||
|
|
+ .setName(HISTOGRAM_RPC_LATENCY)
|
|||
|
|
+ .build();
|
|||
|
|
+- View view = View.builder()
|
|||
|
|
+- .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets))
|
|||
|
|
+- .build();
|
|||
|
|
+- return Lists.newArrayList(new Pair<>(selector, view));
|
|||
|
|
++ ViewBuilder viewBuilder = View.builder()
|
|||
|
|
++ .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets));
|
|||
|
|
++ return Lists.newArrayList(new Pair<>(selector, viewBuilder));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static String getWriteAndFlushResult(Future<?> future) {
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
+index 25e4a166f..6115ead59 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
+@@ -22,7 +22,7 @@ import io.openmessaging.storage.dledger.entry.DLedgerEntry;
|
|||
|
|
+ import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
+ import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+-import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import java.io.File;
|
|||
|
|
+ import java.io.IOException;
|
|||
|
|
+ import java.io.RandomAccessFile;
|
|||
|
|
+@@ -42,23 +42,24 @@ import java.util.Map;
|
|||
|
|
+ import java.util.Objects;
|
|||
|
|
+ import java.util.Optional;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+-import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+-import java.util.concurrent.TimeUnit;
|
|||
|
|
+-import java.util.concurrent.TimeoutException;
|
|||
|
|
+-import java.util.concurrent.ExecutionException;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+-import java.util.concurrent.ExecutorService;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+-import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+-import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentLinkedQueue;
|
|||
|
|
++import java.util.concurrent.ConcurrentMap;
|
|||
|
|
++import java.util.concurrent.ExecutionException;
|
|||
|
|
++import java.util.concurrent.ExecutorService;
|
|||
|
|
++import java.util.concurrent.Executors;
|
|||
|
|
++import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
++import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
++import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
++import java.util.concurrent.TimeUnit;
|
|||
|
|
++import java.util.concurrent.TimeoutException;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicLong;
|
|||
|
|
+ import java.util.function.Supplier;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.AbstractBrokerRunnable;
|
|||
|
|
++import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+ import org.apache.rocketmq.common.BrokerConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.BrokerIdentity;
|
|||
|
|
+ import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+@@ -82,7 +83,6 @@ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.CleanupPolicyUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.QueueTypeUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.ServiceProvider;
|
|||
|
|
+-import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
|
|||
|
|
+@@ -3268,7 +3268,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+- public List<Pair<InstrumentSelector, View>> getMetricsView() {
|
|||
|
|
++ public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {
|
|||
|
|
+ return DefaultStoreMetricsManager.getMetricsView();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
|
|||
|
|
+index 31bbb907f..989cbbe31 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
|
|||
|
|
+@@ -19,8 +19,7 @@ package org.apache.rocketmq.store;
|
|||
|
|
+ import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
+ import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+-import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
+-
|
|||
|
|
++import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+ import java.util.LinkedList;
|
|||
|
|
+@@ -28,7 +27,6 @@ import java.util.List;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.function.Supplier;
|
|||
|
|
+-
|
|||
|
|
+ import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.SystemClock;
|
|||
|
|
+@@ -964,7 +962,7 @@ public interface MessageStore {
|
|||
|
|
+ *
|
|||
|
|
+ * @return List of metrics selector and view pair
|
|||
|
|
+ */
|
|||
|
|
+- List<Pair<InstrumentSelector, View>> getMetricsView();
|
|||
|
|
++ List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Init store metrics
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java
|
|||
|
|
+index ff87f6369..45a6bbc68 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java
|
|||
|
|
+@@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongCounter;
|
|||
|
|
+ import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+ import io.opentelemetry.api.metrics.ObservableLongGauge;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+-import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import java.io.File;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.function.Supplier;
|
|||
|
|
+@@ -69,7 +69,7 @@ public class DefaultStoreMetricsManager {
|
|||
|
|
+ public static LongCounter timerDequeueTotal = new NopLongCounter();
|
|||
|
|
+ public static LongCounter timerEnqueueTotal = new NopLongCounter();
|
|||
|
|
+
|
|||
|
|
+- public static List<Pair<InstrumentSelector, View>> getMetricsView() {
|
|||
|
|
++ public static List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {
|
|||
|
|
+ return Lists.newArrayList();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
|
|||
|
|
+index 25e947512..ab9fc6da7 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
|
|||
|
|
+@@ -20,8 +20,7 @@ package org.apache.rocketmq.store.plugin;
|
|||
|
|
+ import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
+ import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+-import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
+-
|
|||
|
|
++import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+ import java.util.LinkedList;
|
|||
|
|
+@@ -29,7 +28,6 @@ import java.util.List;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.function.Supplier;
|
|||
|
|
+-
|
|||
|
|
+ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.SystemClock;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+@@ -643,7 +641,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+- public List<Pair<InstrumentSelector, View>> getMetricsView() {
|
|||
|
|
++ public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {
|
|||
|
|
+ return next.getMetricsView();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+index ced1fb818..5240ac8e9 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+@@ -21,7 +21,7 @@ import io.opentelemetry.api.common.Attributes;
|
|||
|
|
+ import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
+ import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+-import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+@@ -352,8 +352,8 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+- public List<Pair<InstrumentSelector, View>> getMetricsView() {
|
|||
|
|
+- List<Pair<InstrumentSelector, View>> res = super.getMetricsView();
|
|||
|
|
++ public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {
|
|||
|
|
++ List<Pair<InstrumentSelector, ViewBuilder>> res = super.getMetricsView();
|
|||
|
|
+ res.addAll(TieredStoreMetricsManager.getMetricsView());
|
|||
|
|
+ return res;
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java
|
|||
|
|
+index 3ca0fb614..d8a07f0a7 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java
|
|||
|
|
+@@ -27,6 +27,7 @@ import io.opentelemetry.sdk.metrics.Aggregation;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentType;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.Arrays;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+@@ -101,8 +102,8 @@ public class TieredStoreMetricsManager {
|
|||
|
|
+ public static ObservableLongGauge storageSize = new NopObservableLongGauge();
|
|||
|
|
+ public static ObservableLongGauge storageMessageReserveTime = new NopObservableLongGauge();
|
|||
|
|
+
|
|||
|
|
+- public static List<Pair<InstrumentSelector, View>> getMetricsView() {
|
|||
|
|
+- ArrayList<Pair<InstrumentSelector, View>> res = new ArrayList<>();
|
|||
|
|
++ public static List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {
|
|||
|
|
++ ArrayList<Pair<InstrumentSelector, ViewBuilder>> res = new ArrayList<>();
|
|||
|
|
+
|
|||
|
|
+ InstrumentSelector providerRpcLatencySelector = InstrumentSelector.builder()
|
|||
|
|
+ .setType(InstrumentType.HISTOGRAM)
|
|||
|
|
+@@ -114,10 +115,9 @@ public class TieredStoreMetricsManager {
|
|||
|
|
+ .setName(HISTOGRAM_API_LATENCY)
|
|||
|
|
+ .build();
|
|||
|
|
+
|
|||
|
|
+- View rpcLatencyView = View.builder()
|
|||
|
|
++ ViewBuilder rpcLatencyViewBuilder = View.builder()
|
|||
|
|
+ .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d, 3d, 5d, 7d, 10d, 100d, 200d, 400d, 600d, 800d, 1d * 1000, 1d * 1500, 1d * 3000)))
|
|||
|
|
+- .setDescription("tiered_store_rpc_latency_view")
|
|||
|
|
+- .build();
|
|||
|
|
++ .setDescription("tiered_store_rpc_latency_view");
|
|||
|
|
+
|
|||
|
|
+ InstrumentSelector uploadBufferSizeSelector = InstrumentSelector.builder()
|
|||
|
|
+ .setType(InstrumentType.HISTOGRAM)
|
|||
|
|
+@@ -129,15 +129,14 @@ public class TieredStoreMetricsManager {
|
|||
|
|
+ .setName(HISTOGRAM_DOWNLOAD_BYTES)
|
|||
|
|
+ .build();
|
|||
|
|
+
|
|||
|
|
+- View bufferSizeView = View.builder()
|
|||
|
|
++ ViewBuilder bufferSizeViewBuilder = View.builder()
|
|||
|
|
+ .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d * TieredStoreUtil.KB, 10d * TieredStoreUtil.KB, 100d * TieredStoreUtil.KB, 1d * TieredStoreUtil.MB, 10d * TieredStoreUtil.MB, 32d * TieredStoreUtil.MB, 50d * TieredStoreUtil.MB, 100d * TieredStoreUtil.MB)))
|
|||
|
|
+- .setDescription("tiered_store_buffer_size_view")
|
|||
|
|
+- .build();
|
|||
|
|
++ .setDescription("tiered_store_buffer_size_view");
|
|||
|
|
+
|
|||
|
|
+- res.add(new Pair<>(rpcLatencySelector, rpcLatencyView));
|
|||
|
|
+- res.add(new Pair<>(providerRpcLatencySelector, rpcLatencyView));
|
|||
|
|
+- res.add(new Pair<>(uploadBufferSizeSelector, bufferSizeView));
|
|||
|
|
+- res.add(new Pair<>(downloadBufferSizeSelector, bufferSizeView));
|
|||
|
|
++ res.add(new Pair<>(rpcLatencySelector, rpcLatencyViewBuilder));
|
|||
|
|
++ res.add(new Pair<>(providerRpcLatencySelector, rpcLatencyViewBuilder));
|
|||
|
|
++ res.add(new Pair<>(uploadBufferSizeSelector, bufferSizeViewBuilder));
|
|||
|
|
++ res.add(new Pair<>(downloadBufferSizeSelector, bufferSizeViewBuilder));
|
|||
|
|
+ return res;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From a4bcc2a74d8bec9c9d34565536e87df06e0b11c1 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ziyi Tan <tanziyi0925@gmail.com>
|
|||
|
|
+Date: Thu, 17 Aug 2023 13:53:48 +0800
|
|||
|
|
+Subject: [PATCH 2/7] [ISSUE #7178] refresh metadata after broker startup
|
|||
|
|
+
|
|||
|
|
+Signed-off-by: Ziy1-Tan <ajb459684460@gmail.com>
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/broker/BrokerController.java | 24 +++++++++----------
|
|||
|
|
+ 1 file changed, 12 insertions(+), 12 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+index 30b1d2299..13f9d002b 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+@@ -663,7 +663,7 @@ public class BrokerController {
|
|||
|
|
+ BrokerController.this.getSlaveSynchronize().syncAll();
|
|||
|
|
+ lastSyncTimeMs = System.currentTimeMillis();
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
++
|
|||
|
|
+ //timer checkpoint, latency-sensitive, so sync it more frequently
|
|||
|
|
+ if (messageStoreConfig.isTimerWheelEnable()) {
|
|||
|
|
+ BrokerController.this.getSlaveSynchronize().syncTimerCheckPoint();
|
|||
|
|
+@@ -698,17 +698,6 @@ public class BrokerController {
|
|||
|
|
+
|
|||
|
|
+ initializeBrokerScheduledTasks();
|
|||
|
|
+
|
|||
|
|
+- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
|
|||
|
|
+- @Override
|
|||
|
|
+- public void run() {
|
|||
|
|
+- try {
|
|||
|
|
+- BrokerController.this.brokerOuterAPI.refreshMetadata();
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- LOG.error("ScheduledTask refresh metadata exception", e);
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+- }, 10, 5, TimeUnit.SECONDS);
|
|||
|
|
+-
|
|||
|
|
+ if (this.brokerConfig.getNamesrvAddr() != null) {
|
|||
|
|
+ this.updateNamesrvAddr();
|
|||
|
|
+ LOG.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr());
|
|||
|
|
+@@ -1682,6 +1671,17 @@ public class BrokerController {
|
|||
|
|
+ if (brokerConfig.isSkipPreOnline()) {
|
|||
|
|
+ startServiceWithoutCondition();
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void run() {
|
|||
|
|
++ try {
|
|||
|
|
++ BrokerController.this.brokerOuterAPI.refreshMetadata();
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ LOG.error("ScheduledTask refresh metadata exception", e);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }, 10, 5, TimeUnit.SECONDS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ protected void scheduleSendHeartbeat() {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 3df1b9232af99944cb3d4d4d2d00c5a85cd3b57d Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: guyinyou <36399867+guyinyou@users.noreply.github.com>
|
|||
|
|
+Date: Thu, 17 Aug 2023 13:59:04 +0800
|
|||
|
|
+Subject: [PATCH 3/7] [ISSUE #7201] Remove the DefaultMessageStore.class
|
|||
|
|
+ dependency in TransientStorePool
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: guyinyou <guyinyou.gyy@alibaba-inc.com>
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/store/AllocateMappedFileService.java | 6 +++---
|
|||
|
|
+ .../apache/rocketmq/store/DefaultMessageStore.java | 7 +++++--
|
|||
|
|
+ .../apache/rocketmq/store/TransientStorePool.java | 13 ++++---------
|
|||
|
|
+ 3 files changed, 12 insertions(+), 14 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
|
|||
|
|
+index dca7d5325..c8420fea1 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
|
|||
|
|
+@@ -55,7 +55,7 @@ public class AllocateMappedFileService extends ServiceThread {
|
|||
|
|
+ if (this.messageStore.isTransientStorePoolEnable()) {
|
|||
|
|
+ if (this.messageStore.getMessageStoreConfig().isFastFailIfNoBufferInStorePool()
|
|||
|
|
+ && BrokerRole.SLAVE != this.messageStore.getMessageStoreConfig().getBrokerRole()) { //if broker is slave, don't fast fail even no buffer in pool
|
|||
|
|
+- canSubmitRequests = this.messageStore.getTransientStorePool().availableBufferNums() - this.requestQueue.size();
|
|||
|
|
++ canSubmitRequests = this.messageStore.remainTransientStoreBufferNumbs() - this.requestQueue.size();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -65,7 +65,7 @@ public class AllocateMappedFileService extends ServiceThread {
|
|||
|
|
+ if (nextPutOK) {
|
|||
|
|
+ if (canSubmitRequests <= 0) {
|
|||
|
|
+ log.warn("[NOTIFYME]TransientStorePool is not enough, so create mapped file error, " +
|
|||
|
|
+- "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums());
|
|||
|
|
++ "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs());
|
|||
|
|
+ this.requestTable.remove(nextFilePath);
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+@@ -81,7 +81,7 @@ public class AllocateMappedFileService extends ServiceThread {
|
|||
|
|
+ if (nextNextPutOK) {
|
|||
|
|
+ if (canSubmitRequests <= 0) {
|
|||
|
|
+ log.warn("[NOTIFYME]TransientStorePool is not enough, so skip preallocate mapped file, " +
|
|||
|
|
+- "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums());
|
|||
|
|
++ "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs());
|
|||
|
|
+ this.requestTable.remove(nextNextFilePath);
|
|||
|
|
+ } else {
|
|||
|
|
+ boolean offerOK = this.requestQueue.offer(nextNextReq);
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
+index 6115ead59..f2a54ddf6 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
+@@ -250,7 +250,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
+ this.reputMessageService = new ConcurrentReputMessageService();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- this.transientStorePool = new TransientStorePool(this);
|
|||
|
|
++ this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog());
|
|||
|
|
+
|
|||
|
|
+ this.scheduledExecutorService =
|
|||
|
|
+ Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity()));
|
|||
|
|
+@@ -1983,7 +1983,10 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public int remainTransientStoreBufferNumbs() {
|
|||
|
|
+- return this.transientStorePool.availableBufferNums();
|
|||
|
|
++ if (this.isTransientStorePoolEnable()) {
|
|||
|
|
++ return this.transientStorePool.availableBufferNums();
|
|||
|
|
++ }
|
|||
|
|
++ return Integer.MAX_VALUE;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java
|
|||
|
|
+index 8c1a5338b..0d42ee69e 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java
|
|||
|
|
+@@ -33,13 +33,11 @@ public class TransientStorePool {
|
|||
|
|
+ private final int poolSize;
|
|||
|
|
+ private final int fileSize;
|
|||
|
|
+ private final Deque<ByteBuffer> availableBuffers;
|
|||
|
|
+- private final DefaultMessageStore messageStore;
|
|||
|
|
+ private volatile boolean isRealCommit = true;
|
|||
|
|
+
|
|||
|
|
+- public TransientStorePool(final DefaultMessageStore messageStore) {
|
|||
|
|
+- this.messageStore = messageStore;
|
|||
|
|
+- this.poolSize = messageStore.getMessageStoreConfig().getTransientStorePoolSize();
|
|||
|
|
+- this.fileSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();
|
|||
|
|
++ public TransientStorePool(final int poolSize, final int fileSize) {
|
|||
|
|
++ this.poolSize = poolSize;
|
|||
|
|
++ this.fileSize = fileSize;
|
|||
|
|
+ this.availableBuffers = new ConcurrentLinkedDeque<>();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -81,10 +79,7 @@ public class TransientStorePool {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public int availableBufferNums() {
|
|||
|
|
+- if (messageStore.isTransientStorePoolEnable()) {
|
|||
|
|
+- return availableBuffers.size();
|
|||
|
|
+- }
|
|||
|
|
+- return Integer.MAX_VALUE;
|
|||
|
|
++ return availableBuffers.size();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public boolean isRealCommit() {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 2b93e1e32fd458d9df2091e89ea259ddd4d54061 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: iamgd67 <iamgd67@sina.com>
|
|||
|
|
+Date: Thu, 17 Aug 2023 15:31:14 +0800
|
|||
|
|
+Subject: [PATCH 4/7] Update mqbroker to use runbroker.sh instead of
|
|||
|
|
+ runserver.sh when use --enable-proxy (#7150)
|
|||
|
|
+MIME-Version: 1.0
|
|||
|
|
+Content-Type: text/plain; charset=UTF-8
|
|||
|
|
+Content-Transfer-Encoding: 8bit
|
|||
|
|
+
|
|||
|
|
+Update mqbroker to use runbroker.sh instead of runserver.sh when enabling `--enable-proxy`
|
|||
|
|
+this allow JVM `heap` and `gc` configuration using broker's settings instead of other common serverices'(proxy,namenode, etc).
|
|||
|
|
+our main purpose, like the filename `mqbroker` suggest, is to start broker (which embeds a proxy), so use broker's config is reasonable
|
|||
|
|
+
|
|||
|
|
+chinese version
|
|||
|
|
+mqbroker的--enable-proxy选项是启动内嵌了proxy的broker，而不是内嵌broker的proxy，而且broker的工作量和重要程度大于proxy，所以使用broker的gc和heap配置更合适
|
|||
|
|
+---
|
|||
|
|
+ distribution/bin/mqbroker | 4 ++--
|
|||
|
|
+ 1 file changed, 2 insertions(+), 2 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/distribution/bin/mqbroker b/distribution/bin/mqbroker
|
|||
|
|
+index 3758ed597..35eb93c44 100644
|
|||
|
|
+--- a/distribution/bin/mqbroker
|
|||
|
|
++++ b/distribution/bin/mqbroker
|
|||
|
|
+@@ -68,11 +68,11 @@ if [ "$enable_proxy" = true ]; then
|
|||
|
|
+ if [ "$broker_config" != "" ]; then
|
|||
|
|
+ args_for_proxy=${args_for_proxy}" -bc "${broker_config}
|
|||
|
|
+ fi
|
|||
|
|
+- sh ${ROCKETMQ_HOME}/bin/runserver.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy}
|
|||
|
|
++ sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy}
|
|||
|
|
+ else
|
|||
|
|
+ args_for_broker=$other_args
|
|||
|
|
+ if [ "$broker_config" != "" ]; then
|
|||
|
|
+ args_for_broker=${args_for_broker}" -c "${broker_config}
|
|||
|
|
+ fi
|
|||
|
|
+ sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.broker.logback.xml org.apache.rocketmq.broker.BrokerStartup ${args_for_broker}
|
|||
|
|
+-fi
|
|||
|
|
+\ No newline at end of file
|
|||
|
|
++fi
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 05e7cde610255ed9410fffb0f153efe7c2c8a326 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yao-wenbin <ywb992134@163.com>
|
|||
|
|
+Date: Fri, 18 Aug 2023 09:49:59 +0800
|
|||
|
|
+Subject: [PATCH 5/7] [ISSUE #7042] maven-compile job failed, Because TlsTest's
|
|||
|
|
+ serverRejectsSSLClient test case will throw TooLongFrameException (#7179)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../remoting/netty/NettyRemotingServer.java | 2 +-
|
|||
|
|
+ .../java/org/apache/rocketmq/remoting/TlsTest.java | 14 ++++++++++++--
|
|||
|
|
+ 2 files changed, 13 insertions(+), 3 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
+index 90e358ce3..17f138f86 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
+@@ -502,7 +502,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ case DISABLED:
|
|||
|
|
+ ctx.close();
|
|||
|
|
+ log.warn("Clients intend to establish an SSL connection while this server is running in SSL disabled mode");
|
|||
|
|
+- break;
|
|||
|
|
++ throw new UnsupportedOperationException("The NettyRemotingServer in SSL disabled mode doesn't support ssl client");
|
|||
|
|
+ case PERMISSIVE:
|
|||
|
|
+ case ENFORCING:
|
|||
|
|
+ if (null != sslContext) {
|
|||
|
|
+diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java
|
|||
|
|
+index de7edbbfb..a4890d73d 100644
|
|||
|
|
+--- a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java
|
|||
|
|
++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java
|
|||
|
|
+@@ -144,8 +144,13 @@ public class TlsTest {
|
|||
|
|
+ tlsClientKeyPath = "";
|
|||
|
|
+ tlsClientCertPath = "";
|
|||
|
|
+ clientConfig.setUseTLS(false);
|
|||
|
|
+- } else if ("serverRejectsSSLClient".equals(name.getMethodName())) {
|
|||
|
|
++ } else if ("disabledServerRejectsSSLClient".equals(name.getMethodName())) {
|
|||
|
|
+ tlsMode = TlsMode.DISABLED;
|
|||
|
|
++ } else if ("disabledServerAcceptUnAuthClient".equals(name.getMethodName())) {
|
|||
|
|
++ tlsMode = TlsMode.DISABLED;
|
|||
|
|
++ tlsClientKeyPath = "";
|
|||
|
|
++ tlsClientCertPath = "";
|
|||
|
|
++ clientConfig.setUseTLS(false);
|
|||
|
|
+ } else if ("reloadSslContextForServer".equals(name.getMethodName())) {
|
|||
|
|
+ tlsClientAuthServer = false;
|
|||
|
|
+ tlsServerNeedClientAuth = "none";
|
|||
|
|
+@@ -211,7 +216,7 @@ public class TlsTest {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+- public void serverRejectsSSLClient() throws Exception {
|
|||
|
|
++ public void disabledServerRejectsSSLClient() throws Exception {
|
|||
|
|
+ try {
|
|||
|
|
+ RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 5);
|
|||
|
|
+ failBecauseExceptionWasNotThrown(RemotingSendRequestException.class);
|
|||
|
|
+@@ -219,6 +224,11 @@ public class TlsTest {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Test
|
|||
|
|
++ public void disabledServerAcceptUnAuthClient() throws Exception {
|
|||
|
|
++ requestThenAssertResponse();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ /**
|
|||
|
|
+ * Tests that a server configured to require client authentication refuses to accept connections
|
|||
|
|
+ * from a client that has an untrusted certificate.
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 72d796f2b20b3ec6aebca8c004d9275d7c749a95 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lk <xdkxlk@outlook.com>
|
|||
|
|
+Date: Fri, 18 Aug 2023 11:55:39 +0800
|
|||
|
|
+Subject: [PATCH 6/7] [ISSUE #7205] support batch ack for pop orderly (#7206)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../broker/processor/AckMessageProcessor.java | 99 ++++++-----
|
|||
|
|
+ .../rocketmq/client/impl/MQClientAPIImpl.java | 91 ++++++++--
|
|||
|
|
+ .../test/client/rmq/RMQPopClient.java | 22 +++
|
|||
|
|
+ .../client/consumer/pop/BasePopNormally.java | 6 +
|
|||
|
|
+ .../test/client/consumer/pop/BatchAckIT.java | 159 ++++++++++++++++++
|
|||
|
|
+ 5 files changed, 322 insertions(+), 55 deletions(-)
|
|||
|
|
+ create mode 100644 test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
|
|||
|
|
+index 687811409..244b459d6 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
|
|||
|
|
+@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.processor;
|
|||
|
|
+ import com.alibaba.fastjson.JSON;
|
|||
|
|
+ import io.netty.channel.Channel;
|
|||
|
|
+ import io.netty.channel.ChannelHandlerContext;
|
|||
|
|
++import java.util.BitSet;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+ import org.apache.rocketmq.broker.metrics.PopMetricsManager;
|
|||
|
|
+ import org.apache.rocketmq.common.KeyBuilder;
|
|||
|
|
+@@ -186,46 +187,7 @@ public class AckMessageProcessor implements NettyRequestProcessor {
|
|||
|
|
+ invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfo);
|
|||
|
|
+
|
|||
|
|
+ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) {
|
|||
|
|
+- // order
|
|||
|
|
+- String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId;
|
|||
|
|
+- long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId);
|
|||
|
|
+- if (ackOffset < oldOffset) {
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) {
|
|||
|
|
+- }
|
|||
|
|
+- try {
|
|||
|
|
+- oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId);
|
|||
|
|
+- if (ackOffset < oldOffset) {
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext(
|
|||
|
|
+- topic, consumeGroup,
|
|||
|
|
+- qId, ackOffset,
|
|||
|
|
+- popTime);
|
|||
|
|
+- if (nextOffset > -1) {
|
|||
|
|
+- if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset(
|
|||
|
|
+- topic, consumeGroup, qId)) {
|
|||
|
|
+- this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(),
|
|||
|
|
+- consumeGroup, topic, qId, nextOffset);
|
|||
|
|
+- }
|
|||
|
|
+- if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic,
|
|||
|
|
+- consumeGroup, qId, invisibleTime)) {
|
|||
|
|
+- this.brokerController.getPopMessageProcessor().notifyMessageArriving(
|
|||
|
|
+- topic, consumeGroup, qId);
|
|||
|
|
+- }
|
|||
|
|
+- } else if (nextOffset == -1) {
|
|||
|
|
+- String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s",
|
|||
|
|
+- lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress());
|
|||
|
|
+- POP_LOGGER.warn(errorInfo);
|
|||
|
|
+- response.setCode(ResponseCode.MESSAGE_ILLEGAL);
|
|||
|
|
+- response.setRemark(errorInfo);
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- } finally {
|
|||
|
|
+- this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey);
|
|||
|
|
+- }
|
|||
|
|
+- brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount);
|
|||
|
|
++ ackOrderly(topic, consumeGroup, qId, ackOffset, popTime, invisibleTime, channel, response);
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -250,17 +212,22 @@ public class AckMessageProcessor implements NettyRequestProcessor {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ BatchAckMsg batchAckMsg = new BatchAckMsg();
|
|||
|
|
+- for (int i = 0; batchAck.getBitSet() != null && i < batchAck.getBitSet().length(); i++) {
|
|||
|
|
+- if (!batchAck.getBitSet().get(i)) {
|
|||
|
|
+- continue;
|
|||
|
|
++ BitSet bitSet = batchAck.getBitSet();
|
|||
|
|
++ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
|
|||
|
|
++ if (i == Integer.MAX_VALUE) {
|
|||
|
|
++ break;
|
|||
|
|
+ }
|
|||
|
|
+ long offset = startOffset + i;
|
|||
|
|
+ if (offset < minOffset || offset > maxOffset) {
|
|||
|
|
+ continue;
|
|||
|
|
+ }
|
|||
|
|
+- batchAckMsg.getAckOffsetList().add(offset);
|
|||
|
|
++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) {
|
|||
|
|
++ ackOrderly(topic, consumeGroup, qId, offset, popTime, invisibleTime, channel, response);
|
|||
|
|
++ } else {
|
|||
|
|
++ batchAckMsg.getAckOffsetList().add(offset);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+- if (batchAckMsg.getAckOffsetList().isEmpty()) {
|
|||
|
|
++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE || batchAckMsg.getAckOffsetList().isEmpty()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -311,4 +278,46 @@ public class AckMessageProcessor implements NettyRequestProcessor {
|
|||
|
|
+ PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());
|
|||
|
|
+ brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime, long invisibleTime, Channel channel, RemotingCommand response) {
|
|||
|
|
++ String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId;
|
|||
|
|
++ long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId);
|
|||
|
|
++ if (ackOffset < oldOffset) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) {
|
|||
|
|
++ }
|
|||
|
|
++ try {
|
|||
|
|
++ oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId);
|
|||
|
|
++ if (ackOffset < oldOffset) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext(
|
|||
|
|
++ topic, consumeGroup,
|
|||
|
|
++ qId, ackOffset,
|
|||
|
|
++ popTime);
|
|||
|
|
++ if (nextOffset > -1) {
|
|||
|
|
++ if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset(
|
|||
|
|
++ topic, consumeGroup, qId)) {
|
|||
|
|
++ this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(),
|
|||
|
|
++ consumeGroup, topic, qId, nextOffset);
|
|||
|
|
++ }
|
|||
|
|
++ if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic,
|
|||
|
|
++ consumeGroup, qId, invisibleTime)) {
|
|||
|
|
++ this.brokerController.getPopMessageProcessor().notifyMessageArriving(
|
|||
|
|
++ topic, consumeGroup, qId);
|
|||
|
|
++ }
|
|||
|
|
++ } else if (nextOffset == -1) {
|
|||
|
|
++ String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s",
|
|||
|
|
++ lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress());
|
|||
|
|
++ POP_LOGGER.warn(errorInfo);
|
|||
|
|
++ response.setCode(ResponseCode.MESSAGE_ILLEGAL);
|
|||
|
|
++ response.setRemark(errorInfo);
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ } finally {
|
|||
|
|
++ this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey);
|
|||
|
|
++ }
|
|||
|
|
++ brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, 1);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+index 5101ffc8e..213c26fd6 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+@@ -21,6 +21,7 @@ import java.io.UnsupportedEncodingException;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.Arrays;
|
|||
|
|
++import java.util.BitSet;
|
|||
|
|
+ import java.util.Collections;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+ import java.util.Iterator;
|
|||
|
|
+@@ -54,6 +55,7 @@ import org.apache.rocketmq.client.producer.SendCallback;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.SendResult;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.SendStatus;
|
|||
|
|
+ import org.apache.rocketmq.common.AclConfig;
|
|||
|
|
++import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+ import org.apache.rocketmq.common.MQVersion;
|
|||
|
|
+ import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+@@ -76,7 +78,8 @@ import org.apache.rocketmq.common.namesrv.NameServerUpdateCallback;
|
|||
|
|
+ import org.apache.rocketmq.common.namesrv.TopAddressing;
|
|||
|
|
+ import org.apache.rocketmq.common.sysflag.PullSysFlag;
|
|||
|
|
+ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
+-import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
++import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.CommandCustomHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.InvokeCallback;
|
|||
|
|
+ import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+@@ -101,7 +104,10 @@ import org.apache.rocketmq.remoting.protocol.RequestCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.ResponseCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.body.BatchAck;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo;
|
|||
|
|
+@@ -114,7 +120,6 @@ import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.GroupList;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.KVTable;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.LockBatchResponseBody;
|
|||
|
|
+@@ -196,6 +201,10 @@ import org.apache.rocketmq.remoting.protocol.header.UpdateGlobalWhiteAddrsConfig
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.ViewBrokerStatsDataRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader;
|
|||
|
|
+@@ -207,10 +216,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestH
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.namesrv.PutKVConfigRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
|
|||
|
|
+@@ -221,8 +226,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
|
|||
|
|
+ import org.apache.rocketmq.remoting.rpchook.DynamicalExtFieldRPCHook;
|
|||
|
|
+ import org.apache.rocketmq.remoting.rpchook.StreamTypeRPCHook;
|
|||
|
|
+-import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+
|
|||
|
|
+ import static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS;
|
|||
|
|
+
|
|||
|
|
+@@ -885,9 +888,77 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ final String addr,
|
|||
|
|
+ final long timeOut,
|
|||
|
|
+ final AckCallback ackCallback,
|
|||
|
|
+- final AckMessageRequestHeader requestHeader //
|
|||
|
|
++ final AckMessageRequestHeader requestHeader
|
|||
|
|
++ ) throws RemotingException, MQBrokerException, InterruptedException {
|
|||
|
|
++ ackMessageAsync(addr, timeOut, ackCallback, requestHeader, null);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void batchAckMessageAsync(
|
|||
|
|
++ final String addr,
|
|||
|
|
++ final long timeOut,
|
|||
|
|
++ final AckCallback ackCallback,
|
|||
|
|
++ final String topic,
|
|||
|
|
++ final String consumerGroup,
|
|||
|
|
++ final List<String> extraInfoList
|
|||
|
|
++ ) throws RemotingException, MQBrokerException, InterruptedException {
|
|||
|
|
++ String brokerName = null;
|
|||
|
|
++ Map<String, BatchAck> batchAckMap = new HashMap<>();
|
|||
|
|
++ for (String extraInfo : extraInfoList) {
|
|||
|
|
++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo);
|
|||
|
|
++ if (brokerName == null) {
|
|||
|
|
++ brokerName = ExtraInfoUtil.getBrokerName(extraInfoData);
|
|||
|
|
++ }
|
|||
|
|
++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" +
|
|||
|
|
++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" +
|
|||
|
|
++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" +
|
|||
|
|
++ ExtraInfoUtil.getPopTime(extraInfoData);
|
|||
|
|
++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> {
|
|||
|
|
++ BatchAck newBatchAck = new BatchAck();
|
|||
|
|
++ newBatchAck.setConsumerGroup(consumerGroup);
|
|||
|
|
++ newBatchAck.setTopic(topic);
|
|||
|
|
++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData));
|
|||
|
|
++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData));
|
|||
|
|
++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData));
|
|||
|
|
++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData));
|
|||
|
|
++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData));
|
|||
|
|
++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData));
|
|||
|
|
++ newBatchAck.setBitSet(new BitSet());
|
|||
|
|
++ return newBatchAck;
|
|||
|
|
++ });
|
|||
|
|
++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData)));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody();
|
|||
|
|
++ requestBody.setBrokerName(brokerName);
|
|||
|
|
++ requestBody.setAcks(new ArrayList<>(batchAckMap.values()));
|
|||
|
|
++ batchAckMessageAsync(addr, timeOut, ackCallback, requestBody);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void batchAckMessageAsync(
|
|||
|
|
++ final String addr,
|
|||
|
|
++ final long timeOut,
|
|||
|
|
++ final AckCallback ackCallback,
|
|||
|
|
++ final BatchAckMessageRequestBody requestBody
|
|||
|
|
+ ) throws RemotingException, MQBrokerException, InterruptedException {
|
|||
|
|
+- final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);
|
|||
|
|
++ ackMessageAsync(addr, timeOut, ackCallback, null, requestBody);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected void ackMessageAsync(
|
|||
|
|
++ final String addr,
|
|||
|
|
++ final long timeOut,
|
|||
|
|
++ final AckCallback ackCallback,
|
|||
|
|
++ final AckMessageRequestHeader requestHeader,
|
|||
|
|
++ final BatchAckMessageRequestBody requestBody
|
|||
|
|
++ ) throws RemotingException, MQBrokerException, InterruptedException {
|
|||
|
|
++ RemotingCommand request;
|
|||
|
|
++ if (requestHeader != null) {
|
|||
|
|
++ request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);
|
|||
|
|
++ } else {
|
|||
|
|
++ request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);
|
|||
|
|
++ if (requestBody != null) {
|
|||
|
|
++ request.setBody(requestBody.encode());
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
+ this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java
|
|||
|
|
+index 496bd6da4..09c60c0b4 100644
|
|||
|
|
+--- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java
|
|||
|
|
++++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java
|
|||
|
|
+@@ -17,6 +17,7 @@
|
|||
|
|
+
|
|||
|
|
+ package org.apache.rocketmq.test.client.rmq;
|
|||
|
|
+
|
|||
|
|
++import java.util.List;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import org.apache.rocketmq.client.ClientConfig;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckCallback;
|
|||
|
|
+@@ -140,6 +141,27 @@ public class RMQPopClient implements MQConsumer {
|
|||
|
|
+ return future;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public CompletableFuture<AckResult> batchAckMessageAsync(String brokerAddr, String topic, String consumerGroup,
|
|||
|
|
++ List<String> extraInfoList) {
|
|||
|
|
++ CompletableFuture<AckResult> future = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ this.mqClientAPI.batchAckMessageAsync(brokerAddr, DEFAULT_TIMEOUT, new AckCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onSuccess(AckResult ackResult) {
|
|||
|
|
++ future.complete(ackResult);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onException(Throwable e) {
|
|||
|
|
++ future.completeExceptionally(e);
|
|||
|
|
++ }
|
|||
|
|
++ }, topic, consumerGroup, extraInfoList);
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ return future;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public CompletableFuture<AckResult> changeInvisibleTimeAsync(String brokerAddr, String brokerName, String topic,
|
|||
|
|
+ String consumerGroup, String extraInfo, long invisibleTime) {
|
|||
|
|
+ String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo);
|
|||
|
|
+diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java
|
|||
|
|
+index 952fbe3f5..2e29b95a5 100644
|
|||
|
|
+--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java
|
|||
|
|
++++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java
|
|||
|
|
+@@ -63,4 +63,10 @@ public class BasePopNormally extends BasePop {
|
|||
|
|
+ brokerAddr, messageQueue, invisibleTime, maxNums, group, timeout, true,
|
|||
|
|
+ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*");
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ protected CompletableFuture<PopResult> popMessageAsync(long invisibleTime, int maxNums) {
|
|||
|
|
++ return client.popMessageAsync(
|
|||
|
|
++ brokerAddr, messageQueue, invisibleTime, maxNums, group, 3000, false,
|
|||
|
|
++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*");
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..ec9153ccc
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java
|
|||
|
|
+@@ -0,0 +1,159 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++
|
|||
|
|
++package org.apache.rocketmq.test.client.consumer.pop;
|
|||
|
|
++
|
|||
|
|
++import java.time.Duration;
|
|||
|
|
++import java.util.ArrayList;
|
|||
|
|
++import java.util.List;
|
|||
|
|
++import java.util.concurrent.CompletableFuture;
|
|||
|
|
++import java.util.concurrent.TimeUnit;
|
|||
|
|
++import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
++import java.util.function.Supplier;
|
|||
|
|
++import org.apache.rocketmq.client.consumer.AckResult;
|
|||
|
|
++import org.apache.rocketmq.client.consumer.AckStatus;
|
|||
|
|
++import org.apache.rocketmq.client.consumer.PopResult;
|
|||
|
|
++import org.apache.rocketmq.client.consumer.PopStatus;
|
|||
|
|
++import org.apache.rocketmq.common.attribute.CQType;
|
|||
|
|
++import org.apache.rocketmq.common.attribute.TopicMessageType;
|
|||
|
|
++import org.apache.rocketmq.common.constant.ConsumeInitMode;
|
|||
|
|
++import org.apache.rocketmq.common.filter.ExpressionType;
|
|||
|
|
++import org.apache.rocketmq.common.message.MessageConst;
|
|||
|
|
++import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
++import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
++import org.apache.rocketmq.test.base.IntegrationTestBase;
|
|||
|
|
++import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
|
|||
|
|
++import org.apache.rocketmq.test.client.rmq.RMQPopClient;
|
|||
|
|
++import org.apache.rocketmq.test.util.MQRandomUtils;
|
|||
|
|
++import org.junit.After;
|
|||
|
|
++import org.junit.Before;
|
|||
|
|
++import org.junit.Test;
|
|||
|
|
++
|
|||
|
|
++import static org.awaitility.Awaitility.await;
|
|||
|
|
++import static org.junit.Assert.assertEquals;
|
|||
|
|
++
|
|||
|
|
++public class BatchAckIT extends BasePop {
|
|||
|
|
++
|
|||
|
|
++ protected String topic;
|
|||
|
|
++ protected String group;
|
|||
|
|
++ protected RMQNormalProducer producer = null;
|
|||
|
|
++ protected RMQPopClient client = null;
|
|||
|
|
++ protected String brokerAddr;
|
|||
|
|
++ protected MessageQueue messageQueue;
|
|||
|
|
++
|
|||
|
|
++ @Before
|
|||
|
|
++ public void setUp() {
|
|||
|
|
++ brokerAddr = brokerController1.getBrokerAddr();
|
|||
|
|
++ topic = MQRandomUtils.getRandomTopic();
|
|||
|
|
++ group = initConsumerGroup();
|
|||
|
|
++ IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL);
|
|||
|
|
++ producer = getProducer(NAMESRV_ADDR, topic);
|
|||
|
|
++ client = getRMQPopClient();
|
|||
|
|
++ messageQueue = new MessageQueue(topic, BROKER1_NAME, -1);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @After
|
|||
|
|
++ public void tearDown() {
|
|||
|
|
++ shutdown();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testBatchAckNormallyWithPopBuffer() throws Throwable {
|
|||
|
|
++ brokerController1.getBrokerConfig().setEnablePopBufferMerge(true);
|
|||
|
|
++ brokerController2.getBrokerConfig().setEnablePopBufferMerge(true);
|
|||
|
|
++
|
|||
|
|
++ testBatchAck(() -> {
|
|||
|
|
++ try {
|
|||
|
|
++ return popMessageAsync().get();
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ throw new RuntimeException(e);
|
|||
|
|
++ }
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testBatchAckNormallyWithOutPopBuffer() throws Throwable {
|
|||
|
|
++ brokerController1.getBrokerConfig().setEnablePopBufferMerge(false);
|
|||
|
|
++ brokerController2.getBrokerConfig().setEnablePopBufferMerge(false);
|
|||
|
|
++
|
|||
|
|
++ testBatchAck(() -> {
|
|||
|
|
++ try {
|
|||
|
|
++ return popMessageAsync().get();
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ throw new RuntimeException(e);
|
|||
|
|
++ }
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testBatchAckOrderly() throws Throwable {
|
|||
|
|
++ testBatchAck(() -> {
|
|||
|
|
++ try {
|
|||
|
|
++ return popMessageOrderlyAsync().get();
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ throw new RuntimeException(e);
|
|||
|
|
++ }
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void testBatchAck(Supplier<PopResult> popResultSupplier) throws Throwable {
|
|||
|
|
++ // Send 10 messages but do not ack, let them enter the retry topic
|
|||
|
|
++ producer.send(10);
|
|||
|
|
++ AtomicInteger firstMsgRcvNum = new AtomicInteger();
|
|||
|
|
++ await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> {
|
|||
|
|
++ PopResult popResult = popResultSupplier.get();
|
|||
|
|
++ if (popResult.getPopStatus().equals(PopStatus.FOUND)) {
|
|||
|
|
++ firstMsgRcvNum.addAndGet(popResult.getMsgFoundList().size());
|
|||
|
|
++ }
|
|||
|
|
++ assertEquals(10, firstMsgRcvNum.get());
|
|||
|
|
++ });
|
|||
|
|
++ // sleep 6s, expect messages to enter the retry topic
|
|||
|
|
++ TimeUnit.SECONDS.sleep(6);
|
|||
|
|
++
|
|||
|
|
++ producer.send(20);
|
|||
|
|
++ List<String> extraInfoList = new ArrayList<>();
|
|||
|
|
++ await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> {
|
|||
|
|
++ PopResult popResult = popResultSupplier.get();
|
|||
|
|
++ if (popResult.getPopStatus().equals(PopStatus.FOUND)) {
|
|||
|
|
++ for (MessageExt messageExt : popResult.getMsgFoundList()) {
|
|||
|
|
++ extraInfoList.add(messageExt.getProperty(MessageConst.PROPERTY_POP_CK));
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ assertEquals(30, extraInfoList.size());
|
|||
|
|
++ });
|
|||
|
|
++
|
|||
|
|
++ AckResult ackResult = client.batchAckMessageAsync(brokerAddr, topic, group, extraInfoList).get();
|
|||
|
|
++ assertEquals(AckStatus.OK, ackResult.getStatus());
|
|||
|
|
++
|
|||
|
|
++ // sleep 6s, expected that messages that have been acked will not be re-consumed
|
|||
|
|
++ TimeUnit.SECONDS.sleep(6);
|
|||
|
|
++ PopResult popResult = popResultSupplier.get();
|
|||
|
|
++ assertEquals(PopStatus.POLLING_NOT_FOUND, popResult.getPopStatus());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private CompletableFuture<PopResult> popMessageAsync() {
|
|||
|
|
++ return client.popMessageAsync(
|
|||
|
|
++ brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false,
|
|||
|
|
++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*");
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private CompletableFuture<PopResult> popMessageOrderlyAsync() {
|
|||
|
|
++ return client.popMessageAsync(
|
|||
|
|
++ brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false,
|
|||
|
|
++ ConsumeInitMode.MIN, true, ExpressionType.TAG, "*", null);
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From cc16a1b51216e1e80c22011b8b01e060bb4af8b3 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: rongtong <jinrongtong5@163.com>
|
|||
|
|
+Date: Tue, 22 Aug 2023 10:42:25 +0800
|
|||
|
|
+Subject: [PATCH 7/7] Set table reference the same object for
|
|||
|
|
+ setSubscriptionGroupTable method (#7204)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../broker/subscription/SubscriptionGroupManager.java | 5 +----
|
|||
|
|
+ 1 file changed, 1 insertion(+), 4 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
|
|||
|
|
+index 74e39c0fe..e63b93058 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
|
|||
|
|
+@@ -341,10 +341,7 @@ public class SubscriptionGroupManager extends ConfigManager {
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+ public void setSubscriptionGroupTable(ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable) {
|
|||
|
|
+- this.subscriptionGroupTable.clear();
|
|||
|
|
+- for (String key : subscriptionGroupTable.keySet()) {
|
|||
|
|
+- putSubscriptionGroupConfig(subscriptionGroupTable.get(key));
|
|||
|
|
+- }
|
|||
|
|
++ this.subscriptionGroupTable = subscriptionGroupTable;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public boolean containsSubscriptionGroup(String group) {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch011-backport-optimize-opentelemetry-metric-config.patch b/patch011-backport-optimize-opentelemetry-metric-config.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..b59d7e2c7
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch011-backport-optimize-opentelemetry-metric-config.patch
|
|||
|
|
@@ -0,0 +1,2081 @@
|
|||
|
|
+From 744167bd01fab6821b4d5ae1794dc845153d5156 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ziyi Tan <tanziyi0925@gmail.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 08:32:17 +0800
|
|||
|
|
+Subject: [PATCH 1/7] [ISSUE #7142] Add command `RocksDBConfigToJson` to
|
|||
|
|
+ inspect rocksdb content (#7180)
|
|||
|
|
+
|
|||
|
|
+* feat: add command `RocksDBConfigToJson` to inspect rocksdb content
|
|||
|
|
+
|
|||
|
|
+Signed-off-by: Ziy1-Tan <ajb4596984460@gmail.com>
|
|||
|
|
+
|
|||
|
|
+* refactor: fix style
|
|||
|
|
+
|
|||
|
|
+---------
|
|||
|
|
+
|
|||
|
|
+Signed-off-by: Ziy1-Tan <ajb4596984460@gmail.com>
|
|||
|
|
+Co-authored-by: Ziy1-Tan <ajb4596984460@gmail.com>
|
|||
|
|
+---
|
|||
|
|
+ .../tools/command/MQAdminStartup.java | 2 +
|
|||
|
|
+ .../metadata/RocksDBConfigToJsonCommand.java | 118 ++++++++++++++++++
|
|||
|
|
+ .../metadata/KvConfigToJsonCommandTest.java | 65 ++++++++++
|
|||
|
|
+ 3 files changed, 185 insertions(+)
|
|||
|
|
+ create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+ create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java
|
|||
|
|
+
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
+index 890125ca0..324aa1856 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
+@@ -80,6 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.message.SendMessageCommand;
|
|||
|
|
++import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand;
|
|||
|
|
+@@ -211,6 +212,7 @@ public class MQAdminStartup {
|
|||
|
|
+
|
|||
|
|
+ initCommand(new ClusterListSubCommand());
|
|||
|
|
+ initCommand(new TopicListSubCommand());
|
|||
|
|
++ initCommand(new RocksDBConfigToJsonCommand());
|
|||
|
|
+
|
|||
|
|
+ initCommand(new UpdateKvConfigCommand());
|
|||
|
|
+ initCommand(new DeleteKvConfigCommand());
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..3053f4684
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+@@ -0,0 +1,118 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++package org.apache.rocketmq.tools.command.metadata;
|
|||
|
|
++
|
|||
|
|
++import com.alibaba.fastjson.JSONObject;
|
|||
|
|
++import org.apache.commons.cli.CommandLine;
|
|||
|
|
++import org.apache.commons.cli.Option;
|
|||
|
|
++import org.apache.commons.cli.Options;
|
|||
|
|
++import org.apache.commons.lang3.StringUtils;
|
|||
|
|
++import org.apache.rocketmq.common.config.RocksDBConfigManager;
|
|||
|
|
++import org.apache.rocketmq.common.utils.DataConverter;
|
|||
|
|
++import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
++import org.apache.rocketmq.tools.command.SubCommand;
|
|||
|
|
++import org.apache.rocketmq.tools.command.SubCommandException;
|
|||
|
|
++
|
|||
|
|
++import java.io.File;
|
|||
|
|
++import java.util.HashMap;
|
|||
|
|
++import java.util.Map;
|
|||
|
|
++
|
|||
|
|
++public class RocksDBConfigToJsonCommand implements SubCommand {
|
|||
|
|
++ private static final String TOPICS_JSON_CONFIG = "topics";
|
|||
|
|
++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups";
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public String commandName() {
|
|||
|
|
++ return "rocksDBConfigToJson";
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public String commandDesc() {
|
|||
|
|
++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json";
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public Options buildCommandlineOptions(Options options) {
|
|||
|
|
++ Option pathOption = new Option("p", "path", true,
|
|||
|
|
++ "Absolute path to the metadata directory");
|
|||
|
|
++ pathOption.setRequired(true);
|
|||
|
|
++ options.addOption(pathOption);
|
|||
|
|
++
|
|||
|
|
++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " +
|
|||
|
|
++ "topics/subscriptionGroups");
|
|||
|
|
++ configTypeOption.setRequired(true);
|
|||
|
|
++ options.addOption(configTypeOption);
|
|||
|
|
++
|
|||
|
|
++ return options;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException {
|
|||
|
|
++ String path = commandLine.getOptionValue("path").trim();
|
|||
|
|
++ if (StringUtils.isEmpty(path) || !new File(path).exists()) {
|
|||
|
|
++ System.out.print("Rocksdb path is invalid.\n");
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase();
|
|||
|
|
++
|
|||
|
|
++ final long memTableFlushInterval = 60 * 60 * 1000L;
|
|||
|
|
++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval);
|
|||
|
|
++ try {
|
|||
|
|
++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
++ // for topics.json
|
|||
|
|
++ final Map<String, JSONObject> topicsJsonConfig = new HashMap<>();
|
|||
|
|
++ final Map<String, JSONObject> topicConfigTable = new HashMap<>();
|
|||
|
|
++ boolean isLoad = kvConfigManager.load(path, (key, value) -> {
|
|||
|
|
++ final String topic = new String(key, DataConverter.charset);
|
|||
|
|
++ final String topicConfig = new String(value, DataConverter.charset);
|
|||
|
|
++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig);
|
|||
|
|
++ topicConfigTable.put(topic, jsonObject);
|
|||
|
|
++ });
|
|||
|
|
++
|
|||
|
|
++ if (isLoad) {
|
|||
|
|
++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable));
|
|||
|
|
++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true);
|
|||
|
|
++ System.out.print(topicsJsonStr + "\n");
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
++ // for subscriptionGroup.json
|
|||
|
|
++ final Map<String, JSONObject> subscriptionGroupJsonConfig = new HashMap<>();
|
|||
|
|
++ final Map<String, JSONObject> subscriptionGroupTable = new HashMap<>();
|
|||
|
|
++ boolean isLoad = kvConfigManager.load(path, (key, value) -> {
|
|||
|
|
++ final String subscriptionGroup = new String(key, DataConverter.charset);
|
|||
|
|
++ final String subscriptionGroupConfig = new String(value, DataConverter.charset);
|
|||
|
|
++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig);
|
|||
|
|
++ subscriptionGroupTable.put(subscriptionGroup, jsonObject);
|
|||
|
|
++ });
|
|||
|
|
++
|
|||
|
|
++ if (isLoad) {
|
|||
|
|
++ subscriptionGroupJsonConfig.put("subscriptionGroupTable",
|
|||
|
|
++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable));
|
|||
|
|
++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true);
|
|||
|
|
++ System.out.print(subscriptionGroupJsonStr + "\n");
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ System.out.print("Config type was not recognized, configType=" + configType + "\n");
|
|||
|
|
++ } finally {
|
|||
|
|
++ kvConfigManager.stop();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..b2f66c7b0
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java
|
|||
|
|
+@@ -0,0 +1,65 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++package org.apache.rocketmq.tools.command.metadata;
|
|||
|
|
++
|
|||
|
|
++import org.apache.commons.cli.CommandLine;
|
|||
|
|
++import org.apache.commons.cli.DefaultParser;
|
|||
|
|
++import org.apache.commons.cli.Options;
|
|||
|
|
++import org.apache.rocketmq.srvutil.ServerUtil;
|
|||
|
|
++import org.apache.rocketmq.tools.command.SubCommandException;
|
|||
|
|
++import org.junit.Test;
|
|||
|
|
++
|
|||
|
|
++import java.io.File;
|
|||
|
|
++
|
|||
|
|
++import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
++
|
|||
|
|
++public class KvConfigToJsonCommandTest {
|
|||
|
|
++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/";
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testExecute() throws SubCommandException {
|
|||
|
|
++ {
|
|||
|
|
++ String[] cases = new String[]{"topics", "subscriptionGroups"};
|
|||
|
|
++ for (String c : cases) {
|
|||
|
|
++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand();
|
|||
|
|
++ Options options = ServerUtil.buildCommandlineOptions(new Options());
|
|||
|
|
++ String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c};
|
|||
|
|
++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs,
|
|||
|
|
++ cmd.buildCommandlineOptions(options), new DefaultParser());
|
|||
|
|
++ cmd.execute(commandLine, options, null);
|
|||
|
|
++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c);
|
|||
|
|
++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ // invalid cases
|
|||
|
|
++ {
|
|||
|
|
++ String[][] cases = new String[][]{
|
|||
|
|
++ {"-p " + BASE_PATH + "tmpPath", "-t topics"},
|
|||
|
|
++ {"-p ", "-t topics"},
|
|||
|
|
++ {"-p " + BASE_PATH + "topics", "-t invalid_type"}
|
|||
|
|
++ };
|
|||
|
|
++
|
|||
|
|
++ for (String[] c : cases) {
|
|||
|
|
++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand();
|
|||
|
|
++ Options options = ServerUtil.buildCommandlineOptions(new Options());
|
|||
|
|
++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c,
|
|||
|
|
++ cmd.buildCommandlineOptions(options), new DefaultParser());
|
|||
|
|
++ cmd.execute(commandLine, options, null);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From bdede35db365a49b211cdc249c68b0f60a3df46d Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: mxsm <ljbmxsm@gmail.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 08:34:56 +0800
|
|||
|
|
+Subject: [PATCH 2/7] [ISSUE #7124] Fix the typos in the code comments (#7125)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/broker/processor/ReplyMessageProcessor.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
|
|||
|
|
+index b2db356c8..d3bb048f7 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
|
|||
|
|
+@@ -234,7 +234,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor {
|
|||
|
|
+ } else {
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+ response.setRemark(null);
|
|||
|
|
+- //set to zore to avoid client decoding exception
|
|||
|
|
++ //set to zero to avoid client decoding exception
|
|||
|
|
+ responseHeader.setMsgId("0");
|
|||
|
|
+ responseHeader.setQueueId(queueIdInt);
|
|||
|
|
+ responseHeader.setQueueOffset(0L);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 9bb73b9a38548b99ac5126c40380c3c2e7fc586e Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lizhimins <707364882@qq.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 09:46:27 +0800
|
|||
|
|
+Subject: [PATCH 3/7] [#ISSUE 7222] Bug fix and refactoring of the Indexfile in
|
|||
|
|
+ tiered storage (#7224)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../tieredstore/file/TieredIndexFile.java | 38 +++++++--
|
|||
|
|
+ .../tieredstore/file/TieredIndexFileTest.java | 84 +++++--------------
|
|||
|
|
+ 2 files changed, 52 insertions(+), 70 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java
|
|||
|
|
+index 50beb01ae..eda5e0106 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java
|
|||
|
|
+@@ -16,6 +16,7 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.tieredstore.file;
|
|||
|
|
+
|
|||
|
|
++import com.google.common.annotations.VisibleForTesting;
|
|||
|
|
+ import java.io.File;
|
|||
|
|
+ import java.io.IOException;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+@@ -99,7 +100,7 @@ public class TieredIndexFile {
|
|||
|
|
+ this::doScheduleTask, 10, 10, TimeUnit.SECONDS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private void doScheduleTask() {
|
|||
|
|
++ protected void doScheduleTask() {
|
|||
|
|
+ try {
|
|||
|
|
+ curFileLock.lock();
|
|||
|
|
+ try {
|
|||
|
|
+@@ -145,6 +146,11 @@ public class TieredIndexFile {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @VisibleForTesting
|
|||
|
|
++ public MappedFile getPreMappedFile() {
|
|||
|
|
++ return preMappedFile;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ private void initFile() throws IOException {
|
|||
|
|
+ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize);
|
|||
|
|
+ initIndexFileHeader(curMappedFile);
|
|||
|
|
+@@ -156,19 +162,26 @@ public class TieredIndexFile {
|
|||
|
|
+
|
|||
|
|
+ if (isFileSealed(curMappedFile)) {
|
|||
|
|
+ if (preFileExists) {
|
|||
|
|
+- preFile.delete();
|
|||
|
|
++ if (preFile.delete()) {
|
|||
|
|
++ logger.info("Pre IndexFile deleted success", preFilepath);
|
|||
|
|
++ } else {
|
|||
|
|
++ logger.error("Pre IndexFile deleted failed", preFilepath);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ boolean rename = curMappedFile.renameTo(preFilepath);
|
|||
|
|
+ if (rename) {
|
|||
|
|
+ preMappedFile = curMappedFile;
|
|||
|
|
+ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize);
|
|||
|
|
++ initIndexFileHeader(curMappedFile);
|
|||
|
|
+ preFileExists = true;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ if (preFileExists) {
|
|||
|
|
+ synchronized (TieredIndexFile.class) {
|
|||
|
|
+ if (inflightCompactFuture.isDone()) {
|
|||
|
|
+- inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(new CompactTask(storeConfig, preMappedFile, flatFile), null);
|
|||
|
|
++ inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(
|
|||
|
|
++ new CompactTask(storeConfig, preMappedFile, flatFile), null);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+@@ -261,7 +274,8 @@ public class TieredIndexFile {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public CompletableFuture<List<Pair<Long, ByteBuffer>>> queryAsync(String topic, String key, long beginTime, long endTime) {
|
|||
|
|
++ public CompletableFuture<List<Pair<Long, ByteBuffer>>> queryAsync(String topic, String key, long beginTime,
|
|||
|
|
++ long endTime) {
|
|||
|
|
+ int hashCode = indexKeyHashMethod(buildKey(topic, key));
|
|||
|
|
+ int slotPosition = hashCode % maxHashSlotNum;
|
|||
|
|
+ List<TieredFileSegment> fileSegmentList = flatFile.getFileListByTime(beginTime, endTime);
|
|||
|
|
+@@ -355,7 +369,7 @@ public class TieredIndexFile {
|
|||
|
|
+ private final int fileMaxSize;
|
|||
|
|
+ private MappedFile originFile;
|
|||
|
|
+ private TieredFlatFile fileQueue;
|
|||
|
|
+- private final MappedFile compactFile;
|
|||
|
|
++ private MappedFile compactFile;
|
|||
|
|
+
|
|||
|
|
+ public CompactTask(TieredMessageStoreConfig storeConfig, MappedFile originFile,
|
|||
|
|
+ TieredFlatFile fileQueue) throws IOException {
|
|||
|
|
+@@ -381,6 +395,17 @@ public class TieredIndexFile {
|
|||
|
|
+ } catch (Throwable throwable) {
|
|||
|
|
+ logger.error("TieredIndexFile#compactTask: compact index file failed:", throwable);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ try {
|
|||
|
|
++ if (originFile != null) {
|
|||
|
|
++ originFile.destroy(-1);
|
|||
|
|
++ }
|
|||
|
|
++ if (compactFile != null) {
|
|||
|
|
++ compactFile.destroy(-1);
|
|||
|
|
++ }
|
|||
|
|
++ } catch (Throwable throwable) {
|
|||
|
|
++ logger.error("TieredIndexFile#compactTask: destroy index file failed:", throwable);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void compact() {
|
|||
|
|
+@@ -396,6 +421,8 @@ public class TieredIndexFile {
|
|||
|
|
+ fileQueue.commit(true);
|
|||
|
|
+ compactFile.destroy(-1);
|
|||
|
|
+ originFile.destroy(-1);
|
|||
|
|
++ compactFile = null;
|
|||
|
|
++ originFile = null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildCompactFile() {
|
|||
|
|
+@@ -414,6 +441,7 @@ public class TieredIndexFile {
|
|||
|
|
+ if (slotValue != -1) {
|
|||
|
|
+ int indexTotalSize = 0;
|
|||
|
|
+ int indexPosition = slotValue;
|
|||
|
|
++
|
|||
|
|
+ while (indexPosition >= 0 && indexPosition < maxIndexNum) {
|
|||
|
|
+ int indexOffset = INDEX_FILE_HEADER_SIZE + maxHashSlotNum * INDEX_FILE_HASH_SLOT_SIZE
|
|||
|
|
+ + indexPosition * INDEX_FILE_HASH_ORIGIN_INDEX_SIZE;
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java
|
|||
|
|
+index 7ef49578d..262d6645b 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java
|
|||
|
|
+@@ -19,9 +19,8 @@ package org.apache.rocketmq.tieredstore.file;
|
|||
|
|
+ import com.sun.jna.Platform;
|
|||
|
|
+ import java.io.IOException;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
++import java.time.Duration;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+-import java.util.concurrent.TimeUnit;
|
|||
|
|
+-import org.apache.commons.lang3.SystemUtils;
|
|||
|
|
+ import org.apache.commons.lang3.tuple.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
|
|||
|
|
+@@ -31,9 +30,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+ import org.awaitility.Awaitility;
|
|||
|
|
+ import org.junit.After;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+-import org.junit.Assume;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+-import org.junit.Ignore;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
+ public class TieredIndexFileTest {
|
|||
|
|
+@@ -45,11 +42,12 @@ public class TieredIndexFileTest {
|
|||
|
|
+ @Before
|
|||
|
|
+ public void setUp() {
|
|||
|
|
+ storeConfig = new TieredMessageStoreConfig();
|
|||
|
|
++ storeConfig.setBrokerName("IndexFileBroker");
|
|||
|
|
+ storeConfig.setStorePathRootDir(storePath);
|
|||
|
|
+- storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment");
|
|||
|
|
+- storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2);
|
|||
|
|
+- storeConfig.setTieredStoreIndexFileMaxIndexNum(3);
|
|||
|
|
+- mq = new MessageQueue("TieredIndexFileTest", storeConfig.getBrokerName(), 1);
|
|||
|
|
++ storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment");
|
|||
|
|
++ storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5);
|
|||
|
|
++ storeConfig.setTieredStoreIndexFileMaxIndexNum(20);
|
|||
|
|
++ mq = new MessageQueue("IndexFileTest", storeConfig.getBrokerName(), 1);
|
|||
|
|
+ TieredStoreUtil.getMetadataStore(storeConfig);
|
|||
|
|
+ TieredStoreExecutor.init();
|
|||
|
|
+ }
|
|||
|
|
+@@ -61,77 +59,33 @@ public class TieredIndexFileTest {
|
|||
|
|
+ TieredStoreExecutor.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- @Ignore
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testAppendAndQuery() throws IOException, ClassNotFoundException, NoSuchMethodException {
|
|||
|
|
+ if (Platform.isWindows()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- // skip this test on windows
|
|||
|
|
+- Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS);
|
|||
|
|
+-
|
|||
|
|
+ TieredFileAllocator fileQueueFactory = new TieredFileAllocator(storeConfig);
|
|||
|
|
+ TieredIndexFile indexFile = new TieredIndexFile(fileQueueFactory, storePath);
|
|||
|
|
++
|
|||
|
|
+ indexFile.append(mq, 0, "key3", 3, 300, 1000);
|
|||
|
|
+ indexFile.append(mq, 0, "key2", 2, 200, 1100);
|
|||
|
|
+ indexFile.append(mq, 0, "key1", 1, 100, 1200);
|
|||
|
|
+
|
|||
|
|
+- Awaitility.waitAtMost(5, TimeUnit.SECONDS)
|
|||
|
|
+- .until(() -> {
|
|||
|
|
+- List<Pair<Long, ByteBuffer>> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join();
|
|||
|
|
+- if (indexList.size() != 1) {
|
|||
|
|
+- return false;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- ByteBuffer indexBuffer = indexList.get(0).getValue();
|
|||
|
|
+- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 2, indexBuffer.remaining());
|
|||
|
|
+-
|
|||
|
|
+- Assert.assertEquals(1, indexBuffer.getLong(4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8 + 4));
|
|||
|
|
+-
|
|||
|
|
+- Assert.assertEquals(3, indexBuffer.getLong(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(300, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(0, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8 + 4));
|
|||
|
|
+- return true;
|
|||
|
|
+- });
|
|||
|
|
+-
|
|||
|
|
+- indexFile.append(mq, 0, "key4", 4, 400, 1300);
|
|||
|
|
+- indexFile.append(mq, 0, "key4", 4, 400, 1300);
|
|||
|
|
+- indexFile.append(mq, 0, "key4", 4, 400, 1300);
|
|||
|
|
+-
|
|||
|
|
+- Awaitility.waitAtMost(5, TimeUnit.SECONDS)
|
|||
|
|
+- .until(() -> {
|
|||
|
|
+- List<Pair<Long, ByteBuffer>> indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1300, 1300).join();
|
|||
|
|
+- if (indexList.size() != 1) {
|
|||
|
|
+- return false;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- ByteBuffer indexBuffer = indexList.get(0).getValue();
|
|||
|
|
+- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining());
|
|||
|
|
+- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4));
|
|||
|
|
+- return true;
|
|||
|
|
+- });
|
|||
|
|
+-
|
|||
|
|
+- List<Pair<Long, ByteBuffer>> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1300, 1300).join();
|
|||
|
|
++ // do not do schedule task here
|
|||
|
|
++ TieredStoreExecutor.shutdown();
|
|||
|
|
++ List<Pair<Long, ByteBuffer>> indexList =
|
|||
|
|
++ indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join();
|
|||
|
|
+ Assert.assertEquals(0, indexList.size());
|
|||
|
|
+
|
|||
|
|
+- indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1200, 1300).join();
|
|||
|
|
+- Assert.assertEquals(2, indexList.size());
|
|||
|
|
+-
|
|||
|
|
+- ByteBuffer indexBuffer = indexList.get(0).getValue();
|
|||
|
|
+- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining());
|
|||
|
|
+- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4));
|
|||
|
|
++ // do compaction once
|
|||
|
|
++ TieredStoreExecutor.init();
|
|||
|
|
++ storeConfig.setTieredStoreIndexFileRollingIdleInterval(0);
|
|||
|
|
++ indexFile.doScheduleTask();
|
|||
|
|
++ Awaitility.await().atMost(Duration.ofSeconds(10))
|
|||
|
|
++ .until(() -> !indexFile.getPreMappedFile().getFile().exists());
|
|||
|
|
+
|
|||
|
|
+- indexBuffer = indexList.get(1).getValue();
|
|||
|
|
+- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE, indexBuffer.remaining());
|
|||
|
|
+- Assert.assertEquals(2, indexBuffer.getLong(4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8 + 4));
|
|||
|
|
++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join();
|
|||
|
|
++ Assert.assertEquals(1, indexList.size());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 69c26d3d29cde7b4484ecd112ab9224f9f42bf45 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: guyinyou <36399867+guyinyou@users.noreply.github.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 10:27:52 +0800
|
|||
|
|
+Subject: [PATCH 4/7] [ISSUE #7228] Converge the use of some important
|
|||
|
|
+ variables for some class
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/store/ConsumeQueue.java | 16 ++++++------
|
|||
|
|
+ .../rocketmq/store/MappedFileQueue.java | 26 +++++++++++--------
|
|||
|
|
+ .../store/MultiPathMappedFileQueue.java | 4 +--
|
|||
|
|
+ 3 files changed, 24 insertions(+), 22 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
|
|||
|
|
+index a0b886eb0..56bee2af3 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
|
|||
|
|
+@@ -145,7 +145,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+
|
|||
|
|
+ if (offset >= 0 && size > 0) {
|
|||
|
|
+ mappedFileOffset = i + CQ_STORE_UNIT_SIZE;
|
|||
|
|
+- this.maxPhysicOffset = offset + size;
|
|||
|
|
++ this.setMaxPhysicOffset(offset + size);
|
|||
|
|
+ if (isExtAddr(tagsCode)) {
|
|||
|
|
+ maxExtAddr = tagsCode;
|
|||
|
|
+ }
|
|||
|
|
+@@ -409,7 +409,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+
|
|||
|
|
+ int logicFileSize = this.mappedFileSize;
|
|||
|
|
+
|
|||
|
|
+- this.maxPhysicOffset = phyOffset;
|
|||
|
|
++ this.setMaxPhysicOffset(phyOffset);
|
|||
|
|
+ long maxExtAddr = 1;
|
|||
|
|
+ boolean shouldDeleteFile = false;
|
|||
|
|
+ while (true) {
|
|||
|
|
+@@ -435,7 +435,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+ mappedFile.setWrotePosition(pos);
|
|||
|
|
+ mappedFile.setCommittedPosition(pos);
|
|||
|
|
+ mappedFile.setFlushedPosition(pos);
|
|||
|
|
+- this.maxPhysicOffset = offset + size;
|
|||
|
|
++ this.setMaxPhysicOffset(offset + size);
|
|||
|
|
+ // This maybe not take effect, when not every consume queue has extend file.
|
|||
|
|
+ if (isExtAddr(tagsCode)) {
|
|||
|
|
+ maxExtAddr = tagsCode;
|
|||
|
|
+@@ -453,7 +453,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+ mappedFile.setWrotePosition(pos);
|
|||
|
|
+ mappedFile.setCommittedPosition(pos);
|
|||
|
|
+ mappedFile.setFlushedPosition(pos);
|
|||
|
|
+- this.maxPhysicOffset = offset + size;
|
|||
|
|
++ this.setMaxPhysicOffset(offset + size);
|
|||
|
|
+ if (isExtAddr(tagsCode)) {
|
|||
|
|
+ maxExtAddr = tagsCode;
|
|||
|
|
+ }
|
|||
|
|
+@@ -881,8 +881,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+ private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode,
|
|||
|
|
+ final long cqOffset) {
|
|||
|
|
+
|
|||
|
|
+- if (offset + size <= this.maxPhysicOffset) {
|
|||
|
|
+- log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset);
|
|||
|
|
++ if (offset + size <= this.getMaxPhysicOffset()) {
|
|||
|
|
++ log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", this.getMaxPhysicOffset(), offset);
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -926,7 +926,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+ );
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+- this.maxPhysicOffset = offset + size;
|
|||
|
|
++ this.setMaxPhysicOffset(offset + size);
|
|||
|
|
+ return mappedFile.appendMessage(this.byteBufferIndex.array());
|
|||
|
|
+ }
|
|||
|
|
+ return false;
|
|||
|
|
+@@ -1130,7 +1130,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void destroy() {
|
|||
|
|
+- this.maxPhysicOffset = -1;
|
|||
|
|
++ this.setMaxPhysicOffset(-1);
|
|||
|
|
+ this.minLogicOffset = 0;
|
|||
|
|
+ this.mappedFileQueue.destroy();
|
|||
|
|
+ if (isExtReadEnable()) {
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
|
|||
|
|
+index 0bc70642f..32b90d14f 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
|
|||
|
|
+@@ -285,7 +285,7 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+ if (this.mappedFiles.isEmpty())
|
|||
|
|
+ return 0;
|
|||
|
|
+
|
|||
|
|
+- long committed = this.flushedWhere;
|
|||
|
|
++ long committed = this.getFlushedWhere();
|
|||
|
|
+ if (committed != 0) {
|
|||
|
|
+ MappedFile mappedFile = this.getLastMappedFile(0, false);
|
|||
|
|
+ if (mappedFile != null) {
|
|||
|
|
+@@ -442,11 +442,11 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long remainHowManyDataToCommit() {
|
|||
|
|
+- return getMaxWrotePosition() - committedWhere;
|
|||
|
|
++ return getMaxWrotePosition() - getCommittedWhere();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long remainHowManyDataToFlush() {
|
|||
|
|
+- return getMaxOffset() - flushedWhere;
|
|||
|
|
++ return getMaxOffset() - this.getFlushedWhere();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void deleteLastMappedFile() {
|
|||
|
|
+@@ -616,15 +616,15 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+
|
|||
|
|
+ public boolean flush(final int flushLeastPages) {
|
|||
|
|
+ boolean result = true;
|
|||
|
|
+- MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0);
|
|||
|
|
++ MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0);
|
|||
|
|
+ if (mappedFile != null) {
|
|||
|
|
+ long tmpTimeStamp = mappedFile.getStoreTimestamp();
|
|||
|
|
+ int offset = mappedFile.flush(flushLeastPages);
|
|||
|
|
+ long where = mappedFile.getFileFromOffset() + offset;
|
|||
|
|
+- result = where == this.flushedWhere;
|
|||
|
|
+- this.flushedWhere = where;
|
|||
|
|
++ result = where == this.getFlushedWhere();
|
|||
|
|
++ this.setFlushedWhere(where);
|
|||
|
|
+ if (0 == flushLeastPages) {
|
|||
|
|
+- this.storeTimestamp = tmpTimeStamp;
|
|||
|
|
++ this.setStoreTimestamp(tmpTimeStamp);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -633,12 +633,12 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+
|
|||
|
|
+ public synchronized boolean commit(final int commitLeastPages) {
|
|||
|
|
+ boolean result = true;
|
|||
|
|
+- MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0);
|
|||
|
|
++ MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0);
|
|||
|
|
+ if (mappedFile != null) {
|
|||
|
|
+ int offset = mappedFile.commit(commitLeastPages);
|
|||
|
|
+ long where = mappedFile.getFileFromOffset() + offset;
|
|||
|
|
+- result = where == this.committedWhere;
|
|||
|
|
+- this.committedWhere = where;
|
|||
|
|
++ result = where == this.getCommittedWhere();
|
|||
|
|
++ this.setCommittedWhere(where);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return result;
|
|||
|
|
+@@ -763,7 +763,7 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+ mf.destroy(1000 * 3);
|
|||
|
|
+ }
|
|||
|
|
+ this.mappedFiles.clear();
|
|||
|
|
+- this.flushedWhere = 0;
|
|||
|
|
++ this.setFlushedWhere(0);
|
|||
|
|
+
|
|||
|
|
+ // delete parent directory
|
|||
|
|
+ File file = new File(storePath);
|
|||
|
|
+@@ -848,6 +848,10 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+ return storeTimestamp;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public void setStoreTimestamp(long storeTimestamp) {
|
|||
|
|
++ this.storeTimestamp = storeTimestamp;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public List<MappedFile> getMappedFiles() {
|
|||
|
|
+ return mappedFiles;
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java
|
|||
|
|
+index 8f5af9438..8ff050dfe 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java
|
|||
|
|
+@@ -16,7 +16,6 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.store;
|
|||
|
|
+
|
|||
|
|
+-
|
|||
|
|
+ import java.util.Arrays;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+@@ -113,8 +112,7 @@ public class MultiPathMappedFileQueue extends MappedFileQueue {
|
|||
|
|
+ mf.destroy(1000 * 3);
|
|||
|
|
+ }
|
|||
|
|
+ this.mappedFiles.clear();
|
|||
|
|
+- this.flushedWhere = 0;
|
|||
|
|
+-
|
|||
|
|
++ this.setFlushedWhere(0);
|
|||
|
|
+
|
|||
|
|
+ Set<String> storePathSet = getPaths();
|
|||
|
|
+ storePathSet.addAll(getReadonlyPaths());
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 3884f595949462044c5cb3c236199bc1d7ad2341 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?=
|
|||
|
|
+ <shirenchuang@users.noreply.github.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 11:10:30 +0800
|
|||
|
|
+Subject: [PATCH 5/7] [ISSUE #7149] When creating and updating Topic, there
|
|||
|
|
+ will be problems with permission settings (#7151)
|
|||
|
|
+MIME-Version: 1.0
|
|||
|
|
+Content-Type: text/plain; charset=UTF-8
|
|||
|
|
+Content-Transfer-Encoding: 8bit
|
|||
|
|
+
|
|||
|
|
+* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings
|
|||
|
|
+
|
|||
|
|
+* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings
|
|||
|
|
+
|
|||
|
|
+* [issue#7249]
|
|||
|
|
+
|
|||
|
|
+---------
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: 十真 <shirenchuang.src@cainiao.com>
|
|||
|
|
+---
|
|||
|
|
+ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 3 ++-
|
|||
|
|
+ 1 file changed, 2 insertions(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+index 13f9d002b..e8f943702 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+@@ -1733,7 +1733,8 @@ public class BrokerController {
|
|||
|
|
+ new TopicConfig(topicConfig.getTopicName(),
|
|||
|
|
+ topicConfig.getReadQueueNums(),
|
|||
|
|
+ topicConfig.getWriteQueueNums(),
|
|||
|
|
+- this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag());
|
|||
|
|
++ topicConfig.getPerm()
|
|||
|
|
++ & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag());
|
|||
|
|
+ } else {
|
|||
|
|
+ registerTopicConfig = new TopicConfig(topicConfig);
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 017ad110475e8024585327b44f47e5e97aabc63b Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: echooymxq <echooy.mxq@gmail.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 11:11:42 +0800
|
|||
|
|
+Subject: [PATCH 6/7] [ISSUE #7219] Fix Concurrent modify syncStateSet and Mark
|
|||
|
|
+ synchronizing frequently when shrink. (#7220)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../broker/controller/ReplicasManager.java | 29 ++++++++++---------
|
|||
|
|
+ .../ha/autoswitch/AutoSwitchHAService.java | 21 ++++++++------
|
|||
|
|
+ 2 files changed, 28 insertions(+), 22 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
+index abae7cdb0..37c82e434 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
+@@ -542,7 +542,7 @@ public class ReplicasManager {
|
|||
|
|
+ this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId());
|
|||
|
|
+ this.tempBrokerMetadata.clear();
|
|||
|
|
+ this.brokerControllerId = this.brokerMetadata.getBrokerId();
|
|||
|
|
+- this.haService.setBrokerControllerId(this.brokerControllerId);
|
|||
|
|
++ this.haService.setLocalBrokerId(this.brokerControllerId);
|
|||
|
|
+ return true;
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ LOGGER.error("fail to create metadata file", e);
|
|||
|
|
+@@ -594,7 +594,7 @@ public class ReplicasManager {
|
|||
|
|
+ if (this.brokerMetadata.isLoaded()) {
|
|||
|
|
+ this.registerState = RegisterState.CREATE_METADATA_FILE_DONE;
|
|||
|
|
+ this.brokerControllerId = brokerMetadata.getBrokerId();
|
|||
|
|
+- this.haService.setBrokerControllerId(this.brokerControllerId);
|
|||
|
|
++ this.haService.setLocalBrokerId(this.brokerControllerId);
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ // 2. check if temp metadata exist
|
|||
|
|
+@@ -735,23 +735,26 @@ public class ReplicasManager {
|
|||
|
|
+ if (this.checkSyncStateSetTaskFuture != null) {
|
|||
|
|
+ this.checkSyncStateSetTaskFuture.cancel(false);
|
|||
|
|
+ }
|
|||
|
|
+- this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(() -> {
|
|||
|
|
+- checkSyncStateSetAndDoReport();
|
|||
|
|
+- }, 3 * 1000, this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS);
|
|||
|
|
++ this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000,
|
|||
|
|
++ this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void checkSyncStateSetAndDoReport() {
|
|||
|
|
+- final Set<Long> newSyncStateSet = this.haService.maybeShrinkSyncStateSet();
|
|||
|
|
+- newSyncStateSet.add(this.brokerControllerId);
|
|||
|
|
+- synchronized (this) {
|
|||
|
|
+- if (this.syncStateSet != null) {
|
|||
|
|
+- // Check if syncStateSet changed
|
|||
|
|
+- if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) {
|
|||
|
|
+- return;
|
|||
|
|
++ try {
|
|||
|
|
++ final Set<Long> newSyncStateSet = this.haService.maybeShrinkSyncStateSet();
|
|||
|
|
++ newSyncStateSet.add(this.brokerControllerId);
|
|||
|
|
++ synchronized (this) {
|
|||
|
|
++ if (this.syncStateSet != null) {
|
|||
|
|
++ // Check if syncStateSet changed
|
|||
|
|
++ if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++ doReportSyncStateSetChanged(newSyncStateSet);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ LOGGER.error("Check syncStateSet error", e);
|
|||
|
|
+ }
|
|||
|
|
+- doReportSyncStateSetChanged(newSyncStateSet);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void doReportSyncStateSetChanged(Set<Long> newSyncStateSet) {
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
+index 6dc734e0c..d5393fdca 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
+@@ -41,6 +41,7 @@ import java.nio.channels.SocketChannel;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
++import java.util.Iterator;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+ import java.util.Objects;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+@@ -73,7 +74,7 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
+ private EpochFileCache epochCache;
|
|||
|
|
+ private AutoSwitchHAClient haClient;
|
|||
|
|
+
|
|||
|
|
+- private Long brokerControllerId = null;
|
|||
|
|
++ private Long localBrokerId = null;
|
|||
|
|
+
|
|||
|
|
+ public AutoSwitchHAService() {
|
|||
|
|
+ }
|
|||
|
|
+@@ -287,9 +288,11 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
+
|
|||
|
|
+ // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable,
|
|||
|
|
+ // it means that the broker has not connected.
|
|||
|
|
+- for (Long slaveBrokerId : newSyncStateSet) {
|
|||
|
|
+- if (!this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) {
|
|||
|
|
+- newSyncStateSet.remove(slaveBrokerId);
|
|||
|
|
++ Iterator<Long> iterator = newSyncStateSet.iterator();
|
|||
|
|
++ while (iterator.hasNext()) {
|
|||
|
|
++ Long slaveBrokerId = iterator.next();
|
|||
|
|
++ if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) {
|
|||
|
|
++ iterator.remove();
|
|||
|
|
+ isSyncStateSetChanged = true;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+@@ -419,7 +422,7 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
+ // To avoid the syncStateSet is not consistent with connectionList.
|
|||
|
|
+ // Fix issue: https://github.com/apache/rocketmq/issues/6662
|
|||
|
|
+ for (Long syncId : currentSyncStateSet) {
|
|||
|
|
+- if (!idList.contains(syncId) && this.brokerControllerId != null && !Objects.equals(syncId, this.brokerControllerId)) {
|
|||
|
|
++ if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) {
|
|||
|
|
+ LOGGER.warn("Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.", syncId);
|
|||
|
|
+ // Without check and re-compute, return the confirmOffset's value directly.
|
|||
|
|
+ return this.defaultMessageStore.getConfirmOffsetDirectly();
|
|||
|
|
+@@ -545,12 +548,12 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
+ return this.epochCache.getAllEntries();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public Long getBrokerControllerId() {
|
|||
|
|
+- return brokerControllerId;
|
|||
|
|
++ public Long getLocalBrokerId() {
|
|||
|
|
++ return localBrokerId;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void setBrokerControllerId(Long brokerControllerId) {
|
|||
|
|
+- this.brokerControllerId = brokerControllerId;
|
|||
|
|
++ public void setLocalBrokerId(Long localBrokerId) {
|
|||
|
|
++ this.localBrokerId = localBrokerId;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ class AutoSwitchAcceptSocketService extends AcceptSocketService {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 77e8e54b37c3fc3ea0beffc1ace6f5bf20af10d9 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lk <xdkxlk@outlook.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 15:56:39 +0800
|
|||
|
|
+Subject: [PATCH 7/7] [ISSUE #7223] Support batch ack for grpc client in proxy
|
|||
|
|
+ (#7225)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../client/impl/mqclient/MQClientAPIExt.java | 26 +++
|
|||
|
|
+ .../rocketmq/proxy/config/ProxyConfig.java | 10 +
|
|||
|
|
+ .../grpc/v2/consumer/AckMessageActivity.java | 136 ++++++++---
|
|||
|
|
+ .../proxy/processor/AbstractProcessor.java | 4 +-
|
|||
|
|
+ .../proxy/processor/BatchAckResult.java | 53 +++++
|
|||
|
|
+ .../proxy/processor/ConsumerProcessor.java | 64 +++++
|
|||
|
|
+ .../processor/DefaultMessagingProcessor.java | 7 +
|
|||
|
|
+ .../proxy/processor/MessagingProcessor.java | 18 ++
|
|||
|
|
+ .../message/ClusterMessageService.java | 16 +-
|
|||
|
|
+ .../service/message/LocalMessageService.java | 58 +++++
|
|||
|
|
+ .../proxy/service/message/MessageService.java | 8 +
|
|||
|
|
+ .../service/message/ReceiptHandleMessage.java | 39 ++++
|
|||
|
|
+ .../v2/consumer/AckMessageActivityTest.java | 221 +++++++++++++++---
|
|||
|
|
+ .../proxy/processor/BaseProcessorTest.java | 18 +-
|
|||
|
|
+ .../processor/ConsumerProcessorTest.java | 115 +++++++++
|
|||
|
|
+ .../service/mqclient/MQClientAPIExtTest.java | 12 +
|
|||
|
|
+ 16 files changed, 728 insertions(+), 77 deletions(-)
|
|||
|
|
+ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java
|
|||
|
|
+ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java
|
|||
|
|
+
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
|
|||
|
|
+index fb8f8d11f..d7c8ef8d9 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
|
|||
|
|
+@@ -306,6 +306,32 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ return future;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public CompletableFuture<AckResult> batchAckMessageAsync(
|
|||
|
|
++ String brokerAddr,
|
|||
|
|
++ String topic,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ List<String> extraInfoList,
|
|||
|
|
++ long timeoutMillis
|
|||
|
|
++ ) {
|
|||
|
|
++ CompletableFuture<AckResult> future = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onSuccess(AckResult ackResult) {
|
|||
|
|
++ future.complete(ackResult);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onException(Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ }, topic, consumerGroup, extraInfoList);
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ return future;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public CompletableFuture<AckResult> changeInvisibleTimeAsync(
|
|||
|
|
+ String brokerAddr,
|
|||
|
|
+ String brokerName,
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+index 39caaa0d9..76a243919 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+@@ -250,6 +250,8 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000;
|
|||
|
|
+ private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000;
|
|||
|
|
+
|
|||
|
|
++ private boolean enableBatchAck = false;
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public void initData() {
|
|||
|
|
+ parseDelayLevel();
|
|||
|
|
+@@ -1379,4 +1381,12 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) {
|
|||
|
|
+ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isEnableBatchAck() {
|
|||
|
|
++ return enableBatchAck;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setEnableBatchAck(boolean enableBatchAck) {
|
|||
|
|
++ this.enableBatchAck = enableBatchAck;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java
|
|||
|
|
+index 9a3a77201..97c716c8f 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java
|
|||
|
|
+@@ -31,12 +31,15 @@ import org.apache.rocketmq.client.consumer.AckStatus;
|
|||
|
|
+ import org.apache.rocketmq.common.consumer.ReceiptHandle;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.MessageReceiptHandle;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
++import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;
|
|||
|
|
++import org.apache.rocketmq.proxy.processor.BatchAckResult;
|
|||
|
|
+ import org.apache.rocketmq.proxy.processor.MessagingProcessor;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+
|
|||
|
|
+ public class AckMessageActivity extends AbstractMessingActivity {
|
|||
|
|
+
|
|||
|
|
+@@ -50,60 +53,98 @@ public class AckMessageActivity extends AbstractMessingActivity {
|
|||
|
|
+
|
|||
|
|
+ try {
|
|||
|
|
+ validateTopicAndConsumerGroup(request.getTopic(), request.getGroup());
|
|||
|
|
+-
|
|||
|
|
+- CompletableFuture<AckMessageResultEntry>[] futures = new CompletableFuture[request.getEntriesCount()];
|
|||
|
|
+- for (int i = 0; i < request.getEntriesCount(); i++) {
|
|||
|
|
+- futures[i] = processAckMessage(ctx, request, request.getEntries(i));
|
|||
|
|
++ String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup());
|
|||
|
|
++ String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic());
|
|||
|
|
++ if (ConfigurationManager.getProxyConfig().isEnableBatchAck()) {
|
|||
|
|
++ future = ackMessageInBatch(ctx, group, topic, request);
|
|||
|
|
++ } else {
|
|||
|
|
++ future = ackMessageOneByOne(ctx, group, topic, request);
|
|||
|
|
+ }
|
|||
|
|
+- CompletableFuture.allOf(futures).whenComplete((val, throwable) -> {
|
|||
|
|
+- if (throwable != null) {
|
|||
|
|
+- future.completeExceptionally(throwable);
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ return future;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected CompletableFuture<AckMessageResponse> ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) {
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList = new ArrayList<>(request.getEntriesCount());
|
|||
|
|
+
|
|||
|
|
++ for (AckMessageEntry ackMessageEntry : request.getEntriesList()) {
|
|||
|
|
++ String handleString = getHandleString(ctx, group, request, ackMessageEntry);
|
|||
|
|
++ handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId()));
|
|||
|
|
++ }
|
|||
|
|
++ return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic)
|
|||
|
|
++ .thenApply(batchAckResultList -> {
|
|||
|
|
++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder();
|
|||
|
|
+ Set<Code> responseCodes = new HashSet<>();
|
|||
|
|
+- List<AckMessageResultEntry> entryList = new ArrayList<>();
|
|||
|
|
+- for (CompletableFuture<AckMessageResultEntry> entryFuture : futures) {
|
|||
|
|
+- AckMessageResultEntry entryResult = entryFuture.join();
|
|||
|
|
+- responseCodes.add(entryResult.getStatus().getCode());
|
|||
|
|
+- entryList.add(entryResult);
|
|||
|
|
+- }
|
|||
|
|
+- AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder()
|
|||
|
|
+- .addAllEntries(entryList);
|
|||
|
|
+- if (responseCodes.size() > 1) {
|
|||
|
|
+- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name()));
|
|||
|
|
+- } else if (responseCodes.size() == 1) {
|
|||
|
|
+- Code code = responseCodes.stream().findAny().get();
|
|||
|
|
+- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name()));
|
|||
|
|
+- } else {
|
|||
|
|
+- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty"));
|
|||
|
|
++ for (BatchAckResult batchAckResult : batchAckResultList) {
|
|||
|
|
++ AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult);
|
|||
|
|
++ responseBuilder.addEntries(entry);
|
|||
|
|
++ responseCodes.add(entry.getStatus().getCode());
|
|||
|
|
+ }
|
|||
|
|
+- future.complete(responseBuilder.build());
|
|||
|
|
++ setAckResponseStatus(responseBuilder, responseCodes);
|
|||
|
|
++ return responseBuilder.build();
|
|||
|
|
+ });
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) {
|
|||
|
|
++ ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage();
|
|||
|
|
++ AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder()
|
|||
|
|
++ .setMessageId(handleMessage.getMessageId())
|
|||
|
|
++ .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle());
|
|||
|
|
++ if (batchAckResult.getProxyException() != null) {
|
|||
|
|
++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException()));
|
|||
|
|
++ } else {
|
|||
|
|
++ AckResult ackResult = batchAckResult.getAckResult();
|
|||
|
|
++ if (AckStatus.OK.equals(ackResult.getStatus())) {
|
|||
|
|
++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()));
|
|||
|
|
++ } else {
|
|||
|
|
++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal"));
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+- return future;
|
|||
|
|
++ return resultBuilder.build();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- protected CompletableFuture<AckMessageResultEntry> processAckMessage(ProxyContext ctx, AckMessageRequest request,
|
|||
|
|
++ protected CompletableFuture<AckMessageResponse> ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) {
|
|||
|
|
++ CompletableFuture<AckMessageResponse> resultFuture = new CompletableFuture<>();
|
|||
|
|
++ CompletableFuture<AckMessageResultEntry>[] futures = new CompletableFuture[request.getEntriesCount()];
|
|||
|
|
++ for (int i = 0; i < request.getEntriesCount(); i++) {
|
|||
|
|
++ futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i));
|
|||
|
|
++ }
|
|||
|
|
++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> {
|
|||
|
|
++ if (throwable != null) {
|
|||
|
|
++ resultFuture.completeExceptionally(throwable);
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ Set<Code> responseCodes = new HashSet<>();
|
|||
|
|
++ List<AckMessageResultEntry> entryList = new ArrayList<>();
|
|||
|
|
++ for (CompletableFuture<AckMessageResultEntry> entryFuture : futures) {
|
|||
|
|
++ AckMessageResultEntry entryResult = entryFuture.join();
|
|||
|
|
++ responseCodes.add(entryResult.getStatus().getCode());
|
|||
|
|
++ entryList.add(entryResult);
|
|||
|
|
++ }
|
|||
|
|
++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder()
|
|||
|
|
++ .addAllEntries(entryList);
|
|||
|
|
++ setAckResponseStatus(responseBuilder, responseCodes);
|
|||
|
|
++ resultFuture.complete(responseBuilder.build());
|
|||
|
|
++ });
|
|||
|
|
++ return resultFuture;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected CompletableFuture<AckMessageResultEntry> processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request,
|
|||
|
|
+ AckMessageEntry ackMessageEntry) {
|
|||
|
|
+ CompletableFuture<AckMessageResultEntry> future = new CompletableFuture<>();
|
|||
|
|
+
|
|||
|
|
+ try {
|
|||
|
|
+- String handleString = ackMessageEntry.getReceiptHandle();
|
|||
|
|
+-
|
|||
|
|
+- String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup());
|
|||
|
|
+- MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle());
|
|||
|
|
+- if (messageReceiptHandle != null) {
|
|||
|
|
+- handleString = messageReceiptHandle.getReceiptHandleStr();
|
|||
|
|
+- }
|
|||
|
|
++ String handleString = this.getHandleString(ctx, group, request, ackMessageEntry);
|
|||
|
|
+ CompletableFuture<AckResult> ackResultFuture = this.messagingProcessor.ackMessage(
|
|||
|
|
+ ctx,
|
|||
|
|
+ ReceiptHandle.decode(handleString),
|
|||
|
|
+ ackMessageEntry.getMessageId(),
|
|||
|
|
+ group,
|
|||
|
|
+- GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()));
|
|||
|
|
++ topic
|
|||
|
|
++ );
|
|||
|
|
+ ackResultFuture.thenAccept(result -> {
|
|||
|
|
+ future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result));
|
|||
|
|
+ }).exceptionally(t -> {
|
|||
|
|
+@@ -139,4 +180,25 @@ public class AckMessageActivity extends AbstractMessingActivity {
|
|||
|
|
+ .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal"))
|
|||
|
|
+ .build();
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set<Code> responseCodes) {
|
|||
|
|
++ if (responseCodes.size() > 1) {
|
|||
|
|
++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name()));
|
|||
|
|
++ } else if (responseCodes.size() == 1) {
|
|||
|
|
++ Code code = responseCodes.stream().findAny().get();
|
|||
|
|
++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name()));
|
|||
|
|
++ } else {
|
|||
|
|
++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty"));
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) {
|
|||
|
|
++ String handleString = ackMessageEntry.getReceiptHandle();
|
|||
|
|
++
|
|||
|
|
++ MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle());
|
|||
|
|
++ if (messageReceiptHandle != null) {
|
|||
|
|
++ handleString = messageReceiptHandle.getReceiptHandleStr();
|
|||
|
|
++ }
|
|||
|
|
++ return handleString;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java
|
|||
|
|
+index b61c3df9e..c63212c23 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java
|
|||
|
|
+@@ -27,6 +27,8 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown {
|
|||
|
|
+ protected MessagingProcessor messagingProcessor;
|
|||
|
|
+ protected ServiceManager serviceManager;
|
|||
|
|
+
|
|||
|
|
++ protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired");
|
|||
|
|
++
|
|||
|
|
+ public AbstractProcessor(MessagingProcessor messagingProcessor,
|
|||
|
|
+ ServiceManager serviceManager) {
|
|||
|
|
+ this.messagingProcessor = messagingProcessor;
|
|||
|
|
+@@ -35,7 +37,7 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown {
|
|||
|
|
+
|
|||
|
|
+ protected void validateReceiptHandle(ReceiptHandle handle) {
|
|||
|
|
+ if (handle.isExpired()) {
|
|||
|
|
+- throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired");
|
|||
|
|
++ throw EXPIRED_HANDLE_PROXY_EXCEPTION;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..dfb9c9b9e
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java
|
|||
|
|
+@@ -0,0 +1,53 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++
|
|||
|
|
++package org.apache.rocketmq.proxy.processor;
|
|||
|
|
++
|
|||
|
|
++import org.apache.rocketmq.client.consumer.AckResult;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.ProxyException;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
++
|
|||
|
|
++public class BatchAckResult {
|
|||
|
|
++
|
|||
|
|
++ private final ReceiptHandleMessage receiptHandleMessage;
|
|||
|
|
++ private AckResult ackResult;
|
|||
|
|
++ private ProxyException proxyException;
|
|||
|
|
++
|
|||
|
|
++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage,
|
|||
|
|
++ AckResult ackResult) {
|
|||
|
|
++ this.receiptHandleMessage = receiptHandleMessage;
|
|||
|
|
++ this.ackResult = ackResult;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage,
|
|||
|
|
++ ProxyException proxyException) {
|
|||
|
|
++ this.receiptHandleMessage = receiptHandleMessage;
|
|||
|
|
++ this.proxyException = proxyException;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public ReceiptHandleMessage getReceiptHandleMessage() {
|
|||
|
|
++ return receiptHandleMessage;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public AckResult getAckResult() {
|
|||
|
|
++ return ackResult;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public ProxyException getProxyException() {
|
|||
|
|
++ return proxyException;
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java
|
|||
|
|
+index 656a6339d..f3522b374 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java
|
|||
|
|
+@@ -48,6 +48,7 @@ import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.FutureUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.ProxyUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.ServiceManager;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
|
|||
|
|
+@@ -241,6 +242,69 @@ public class ConsumerProcessor extends AbstractProcessor {
|
|||
|
|
+ return FutureUtils.addExecutor(future, this.executor);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public CompletableFuture<List<BatchAckResult>> batchAckMessage(
|
|||
|
|
++ ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ String topic,
|
|||
|
|
++ long timeoutMillis
|
|||
|
|
++ ) {
|
|||
|
|
++ CompletableFuture<List<BatchAckResult>> future = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ List<BatchAckResult> batchAckResultList = new ArrayList<>(handleMessageList.size());
|
|||
|
|
++ Map<String, List<ReceiptHandleMessage>> brokerHandleListMap = new HashMap<>();
|
|||
|
|
++
|
|||
|
|
++ for (ReceiptHandleMessage handleMessage : handleMessageList) {
|
|||
|
|
++ if (handleMessage.getReceiptHandle().isExpired()) {
|
|||
|
|
++ batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION));
|
|||
|
|
++ continue;
|
|||
|
|
++ }
|
|||
|
|
++ List<ReceiptHandleMessage> brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>());
|
|||
|
|
++ brokerHandleList.add(handleMessage);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ if (brokerHandleListMap.isEmpty()) {
|
|||
|
|
++ return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor);
|
|||
|
|
++ }
|
|||
|
|
++ Set<Map.Entry<String, List<ReceiptHandleMessage>>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet();
|
|||
|
|
++ CompletableFuture<List<BatchAckResult>>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()];
|
|||
|
|
++ int futureIndex = 0;
|
|||
|
|
++ for (Map.Entry<String, List<ReceiptHandleMessage>> entry : brokerHandleListMapEntrySet) {
|
|||
|
|
++ futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis);
|
|||
|
|
++ }
|
|||
|
|
++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> {
|
|||
|
|
++ if (throwable != null) {
|
|||
|
|
++ future.completeExceptionally(throwable);
|
|||
|
|
++ }
|
|||
|
|
++ for (CompletableFuture<List<BatchAckResult>> resultFuture : futures) {
|
|||
|
|
++ batchAckResultList.addAll(resultFuture.join());
|
|||
|
|
++ }
|
|||
|
|
++ future.complete(batchAckResultList);
|
|||
|
|
++ });
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ return FutureUtils.addExecutor(future, this.executor);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected CompletableFuture<List<BatchAckResult>> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List<ReceiptHandleMessage> handleMessageList, long timeoutMillis) {
|
|||
|
|
++ return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis)
|
|||
|
|
++ .thenApply(result -> {
|
|||
|
|
++ List<BatchAckResult> results = new ArrayList<>();
|
|||
|
|
++ for (ReceiptHandleMessage handleMessage : handleMessageList) {
|
|||
|
|
++ results.add(new BatchAckResult(handleMessage, result));
|
|||
|
|
++ }
|
|||
|
|
++ return results;
|
|||
|
|
++ })
|
|||
|
|
++ .exceptionally(throwable -> {
|
|||
|
|
++ List<BatchAckResult> results = new ArrayList<>();
|
|||
|
|
++ for (ReceiptHandleMessage handleMessage : handleMessageList) {
|
|||
|
|
++ results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable)));
|
|||
|
|
++ }
|
|||
|
|
++ return results;
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public CompletableFuture<AckResult> changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle,
|
|||
|
|
+ String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) {
|
|||
|
|
+ CompletableFuture<AckResult> future = new CompletableFuture<>();
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
|
|||
|
|
+index 188cb7b9b..ba150051b 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
|
|||
|
|
+@@ -46,6 +46,7 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ProxyConfig;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.ServiceManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.ServiceManagerFactory;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.metadata.MetadataService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;
|
|||
|
|
+@@ -183,6 +184,12 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen
|
|||
|
|
+ return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public CompletableFuture<List<BatchAckResult>> batchAckMessage(ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList, String consumerGroup, String topic, long timeoutMillis) {
|
|||
|
|
++ return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<AckResult> changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId,
|
|||
|
|
+ String groupName, String topicName, long invisibleTime, long timeoutMillis) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
|
|||
|
|
+index d86be0bd8..2ae7418ba 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
|
|||
|
|
+@@ -37,6 +37,7 @@ import org.apache.rocketmq.proxy.common.Address;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.MessageReceiptHandle;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.StartAndShutdown;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.metadata.MetadataService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;
|
|||
|
|
+@@ -155,6 +156,23 @@ public interface MessagingProcessor extends StartAndShutdown {
|
|||
|
|
+ long timeoutMillis
|
|||
|
|
+ );
|
|||
|
|
+
|
|||
|
|
++ default CompletableFuture<List<BatchAckResult>> batchAckMessage(
|
|||
|
|
++ ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ String topic
|
|||
|
|
++ ) {
|
|||
|
|
++ return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ CompletableFuture<List<BatchAckResult>> batchAckMessage(
|
|||
|
|
++ ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ String topic,
|
|||
|
|
++ long timeoutMillis
|
|||
|
|
++ );
|
|||
|
|
++
|
|||
|
|
+ default CompletableFuture<AckResult> changeInvisibleTime(
|
|||
|
|
+ ProxyContext ctx,
|
|||
|
|
+ ReceiptHandle handle,
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
|
|||
|
|
+index 9f163f1b9..70b72deae 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
|
|||
|
|
+@@ -20,9 +20,11 @@ import com.google.common.collect.Lists;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
++import java.util.stream.Collectors;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckResult;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.PopResult;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.PullResult;
|
|||
|
|
++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.SendResult;
|
|||
|
|
+ import org.apache.rocketmq.common.consumer.ReceiptHandle;
|
|||
|
|
+ import org.apache.rocketmq.common.message.Message;
|
|||
|
|
+@@ -31,7 +33,6 @@ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyException;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.FutureUtils;
|
|||
|
|
+-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.TopicRouteService;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+@@ -137,6 +138,19 @@ public class ClusterMessageService implements MessageService {
|
|||
|
|
+ );
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public CompletableFuture<AckResult> batchAckMessage(ProxyContext ctx, List<ReceiptHandleMessage> handleList, String consumerGroup,
|
|||
|
|
++ String topic, long timeoutMillis) {
|
|||
|
|
++ List<String> extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList());
|
|||
|
|
++ return this.mqClientAPIFactory.getClient().batchAckMessageAsync(
|
|||
|
|
++ this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()),
|
|||
|
|
++ topic,
|
|||
|
|
++ consumerGroup,
|
|||
|
|
++ extraInfoList,
|
|||
|
|
++ timeoutMillis
|
|||
|
|
++ );
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<PullResult> pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,
|
|||
|
|
+ PullMessageRequestHeader requestHeader, long timeoutMillis) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
|
|||
|
|
+index eb2c4d9ee..ca7dcc9eb 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
|
|||
|
|
+@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.service.message;
|
|||
|
|
+ import io.netty.channel.ChannelHandlerContext;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
++import java.util.BitSet;
|
|||
|
|
+ import java.util.Collections;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+@@ -54,6 +55,8 @@ import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RequestCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.ResponseCode;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.body.BatchAck;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;
|
|||
|
|
+@@ -364,6 +367,61 @@ public class LocalMessageService implements MessageService {
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public CompletableFuture<AckResult> batchAckMessage(ProxyContext ctx, List<ReceiptHandleMessage> handleList,
|
|||
|
|
++ String consumerGroup, String topic, long timeoutMillis) {
|
|||
|
|
++ SimpleChannel channel = channelManager.createChannel(ctx);
|
|||
|
|
++ ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext();
|
|||
|
|
++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);
|
|||
|
|
++
|
|||
|
|
++ Map<String, BatchAck> batchAckMap = new HashMap<>();
|
|||
|
|
++ for (ReceiptHandleMessage receiptHandleMessage : handleList) {
|
|||
|
|
++ String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle();
|
|||
|
|
++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo);
|
|||
|
|
++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" +
|
|||
|
|
++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" +
|
|||
|
|
++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" +
|
|||
|
|
++ ExtraInfoUtil.getPopTime(extraInfoData);
|
|||
|
|
++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> {
|
|||
|
|
++ BatchAck newBatchAck = new BatchAck();
|
|||
|
|
++ newBatchAck.setConsumerGroup(consumerGroup);
|
|||
|
|
++ newBatchAck.setTopic(topic);
|
|||
|
|
++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData));
|
|||
|
|
++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData));
|
|||
|
|
++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData));
|
|||
|
|
++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData));
|
|||
|
|
++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData));
|
|||
|
|
++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData));
|
|||
|
|
++ newBatchAck.setBitSet(new BitSet());
|
|||
|
|
++ return newBatchAck;
|
|||
|
|
++ });
|
|||
|
|
++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData)));
|
|||
|
|
++ }
|
|||
|
|
++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody();
|
|||
|
|
++ requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName());
|
|||
|
|
++ requestBody.setAcks(new ArrayList<>(batchAckMap.values()));
|
|||
|
|
++
|
|||
|
|
++ command.setBody(requestBody.encode());
|
|||
|
|
++ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ RemotingCommand response = brokerController.getAckMessageProcessor()
|
|||
|
|
++ .processRequest(channelHandlerContext, command);
|
|||
|
|
++ future.complete(response);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ log.error("Fail to process batchAckMessage command", e);
|
|||
|
|
++ future.completeExceptionally(e);
|
|||
|
|
++ }
|
|||
|
|
++ return future.thenApply(r -> {
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ if (ResponseCode.SUCCESS == r.getCode()) {
|
|||
|
|
++ ackResult.setStatus(AckStatus.OK);
|
|||
|
|
++ } else {
|
|||
|
|
++ ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
++ }
|
|||
|
|
++ return ackResult;
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<PullResult> pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,
|
|||
|
|
+ PullMessageRequestHeader requestHeader, long timeoutMillis) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
|
|||
|
|
+index 15da17154..58a835adb 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
|
|||
|
|
+@@ -91,6 +91,14 @@ public interface MessageService {
|
|||
|
|
+ long timeoutMillis
|
|||
|
|
+ );
|
|||
|
|
+
|
|||
|
|
++ CompletableFuture<AckResult> batchAckMessage(
|
|||
|
|
++ ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleList,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ String topic,
|
|||
|
|
++ long timeoutMillis
|
|||
|
|
++ );
|
|||
|
|
++
|
|||
|
|
+ CompletableFuture<PullResult> pullMessage(
|
|||
|
|
+ ProxyContext ctx,
|
|||
|
|
+ AddressableMessageQueue messageQueue,
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..ae63fed49
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java
|
|||
|
|
+@@ -0,0 +1,39 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++
|
|||
|
|
++package org.apache.rocketmq.proxy.service.message;
|
|||
|
|
++
|
|||
|
|
++import org.apache.rocketmq.common.consumer.ReceiptHandle;
|
|||
|
|
++
|
|||
|
|
++public class ReceiptHandleMessage {
|
|||
|
|
++
|
|||
|
|
++ private final ReceiptHandle receiptHandle;
|
|||
|
|
++ private final String messageId;
|
|||
|
|
++
|
|||
|
|
++ public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) {
|
|||
|
|
++ this.receiptHandle = receiptHandle;
|
|||
|
|
++ this.messageId = messageId;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public ReceiptHandle getReceiptHandle() {
|
|||
|
|
++ return receiptHandle;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public String getMessageId() {
|
|||
|
|
++ return messageId;
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java
|
|||
|
|
+index 49fdfc6a8..3c4746105 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java
|
|||
|
|
+@@ -20,21 +20,32 @@ package org.apache.rocketmq.proxy.grpc.v2.consumer;
|
|||
|
|
+ import apache.rocketmq.v2.AckMessageEntry;
|
|||
|
|
+ import apache.rocketmq.v2.AckMessageRequest;
|
|||
|
|
+ import apache.rocketmq.v2.AckMessageResponse;
|
|||
|
|
++import apache.rocketmq.v2.AckMessageResultEntry;
|
|||
|
|
+ import apache.rocketmq.v2.Code;
|
|||
|
|
+ import apache.rocketmq.v2.Resource;
|
|||
|
|
++import java.util.ArrayList;
|
|||
|
|
++import java.util.HashMap;
|
|||
|
|
++import java.util.List;
|
|||
|
|
++import java.util.Map;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckResult;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckStatus;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyException;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
|
|||
|
|
++import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;
|
|||
|
|
++import org.apache.rocketmq.proxy.processor.BatchAckResult;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
++import org.mockito.stubbing.Answer;
|
|||
|
|
+
|
|||
|
|
+ import static org.junit.Assert.assertEquals;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.any;
|
|||
|
|
++import static org.mockito.ArgumentMatchers.anyList;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyString;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.eq;
|
|||
|
|
++import static org.mockito.Mockito.doAnswer;
|
|||
|
|
+ import static org.mockito.Mockito.when;
|
|||
|
|
+
|
|||
|
|
+ public class AckMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+@@ -52,43 +63,197 @@ public class AckMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testAckMessage() throws Throwable {
|
|||
|
|
+- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString()))
|
|||
|
|
++ ConfigurationManager.getProxyConfig().setEnableBatchAck(false);
|
|||
|
|
++
|
|||
|
|
++ String msg1 = "msg1";
|
|||
|
|
++ String msg2 = "msg2";
|
|||
|
|
++ String msg3 = "msg3";
|
|||
|
|
++
|
|||
|
|
++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString()))
|
|||
|
|
+ .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"));
|
|||
|
|
+
|
|||
|
|
+ AckResult msg2AckResult = new AckResult();
|
|||
|
|
+ msg2AckResult.setStatus(AckStatus.OK);
|
|||
|
|
+- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString()))
|
|||
|
|
++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString()))
|
|||
|
|
+ .thenReturn(CompletableFuture.completedFuture(msg2AckResult));
|
|||
|
|
+
|
|||
|
|
+ AckResult msg3AckResult = new AckResult();
|
|||
|
|
+ msg3AckResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
+- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString()))
|
|||
|
|
++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString()))
|
|||
|
|
+ .thenReturn(CompletableFuture.completedFuture(msg3AckResult));
|
|||
|
|
+
|
|||
|
|
+- AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
+- createContext(),
|
|||
|
|
+- AckMessageRequest.newBuilder()
|
|||
|
|
+- .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
+- .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
+- .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
+- .setMessageId("msg1")
|
|||
|
|
+- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
+- .build())
|
|||
|
|
+- .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
+- .setMessageId("msg2")
|
|||
|
|
+- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
+- .build())
|
|||
|
|
+- .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
+- .setMessageId("msg3")
|
|||
|
|
+- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
+- .build())
|
|||
|
|
+- .build()
|
|||
|
|
+- ).get();
|
|||
|
|
+-
|
|||
|
|
+- assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode());
|
|||
|
|
+- assertEquals(3, response.getEntriesCount());
|
|||
|
|
+- assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode());
|
|||
|
|
+- assertEquals(Code.OK, response.getEntries(1).getStatus().getCode());
|
|||
|
|
+- assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode());
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg1)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg2)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.OK, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg3)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg1)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg2)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg3)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++
|
|||
|
|
++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode());
|
|||
|
|
++ assertEquals(3, response.getEntriesCount());
|
|||
|
|
++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode());
|
|||
|
|
++ assertEquals(Code.OK, response.getEntries(1).getStatus().getCode());
|
|||
|
|
++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testAckMessageInBatch() throws Throwable {
|
|||
|
|
++ ConfigurationManager.getProxyConfig().setEnableBatchAck(true);
|
|||
|
|
++
|
|||
|
|
++ String successMessageId = "msg1";
|
|||
|
|
++ String notOkMessageId = "msg2";
|
|||
|
|
++ String exceptionMessageId = "msg3";
|
|||
|
|
++
|
|||
|
|
++ doAnswer((Answer<CompletableFuture<List<BatchAckResult>>>) invocation -> {
|
|||
|
|
++ List<ReceiptHandleMessage> receiptHandleMessageList = invocation.getArgument(1, List.class);
|
|||
|
|
++ List<BatchAckResult> batchAckResultList = new ArrayList<>();
|
|||
|
|
++ for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) {
|
|||
|
|
++ BatchAckResult batchAckResult;
|
|||
|
|
++ if (receiptHandleMessage.getMessageId().equals(successMessageId)) {
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ ackResult.setStatus(AckStatus.OK);
|
|||
|
|
++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult);
|
|||
|
|
++ } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) {
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult);
|
|||
|
|
++ } else {
|
|||
|
|
++ batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, ""));
|
|||
|
|
++ }
|
|||
|
|
++ batchAckResultList.add(batchAckResult);
|
|||
|
|
++ }
|
|||
|
|
++ return CompletableFuture.completedFuture(batchAckResultList);
|
|||
|
|
++ }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString());
|
|||
|
|
++
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(successMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.OK, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(notOkMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(exceptionMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(successMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(notOkMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(exceptionMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++
|
|||
|
|
++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode());
|
|||
|
|
++ assertEquals(3, response.getEntriesCount());
|
|||
|
|
++ Map<String, Code> msgCode = new HashMap<>();
|
|||
|
|
++ for (AckMessageResultEntry entry : response.getEntriesList()) {
|
|||
|
|
++ msgCode.put(entry.getMessageId(), entry.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ assertEquals(Code.OK, msgCode.get(successMessageId));
|
|||
|
|
++ assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId));
|
|||
|
|
++ assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId));
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+\ No newline at end of file
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java
|
|||
|
|
+index 5c1ea9627..072630e39 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java
|
|||
|
|
+@@ -66,14 +66,6 @@ public class BaseProcessorTest extends InitConfigTest {
|
|||
|
|
+ protected ProxyRelayService proxyRelayService;
|
|||
|
|
+ @Mock
|
|||
|
|
+ protected MetadataService metadataService;
|
|||
|
|
+- @Mock
|
|||
|
|
+- protected ProducerProcessor producerProcessor;
|
|||
|
|
+- @Mock
|
|||
|
|
+- protected ConsumerProcessor consumerProcessor;
|
|||
|
|
+- @Mock
|
|||
|
|
+- protected TransactionProcessor transactionProcessor;
|
|||
|
|
+- @Mock
|
|||
|
|
+- protected ClientProcessor clientProcessor;
|
|||
|
|
+
|
|||
|
|
+ public void before() throws Throwable {
|
|||
|
|
+ super.before();
|
|||
|
|
+@@ -92,6 +84,13 @@ public class BaseProcessorTest extends InitConfigTest {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) {
|
|||
|
|
++ return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(),
|
|||
|
|
++ RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE),
|
|||
|
|
++ RANDOM.nextInt(Integer.MAX_VALUE), "mockBroker");
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime,
|
|||
|
|
++ long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) {
|
|||
|
|
+ MessageExt messageExt = new MessageExt();
|
|||
|
|
+ messageExt.setTopic(topic);
|
|||
|
|
+ messageExt.setTags(tags);
|
|||
|
|
+@@ -100,8 +99,7 @@ public class BaseProcessorTest extends InitConfigTest {
|
|||
|
|
+ messageExt.setMsgId(MessageClientIDSetter.createUniqID());
|
|||
|
|
+ messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE));
|
|||
|
|
+ MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK,
|
|||
|
|
+- ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime,
|
|||
|
|
+- RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE)));
|
|||
|
|
++ ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset));
|
|||
|
|
+ return messageExt;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java
|
|||
|
|
+index 717e86fc0..db268a06e 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java
|
|||
|
|
+@@ -20,8 +20,11 @@ package org.apache.rocketmq.proxy.processor;
|
|||
|
|
+ import com.google.common.collect.Sets;
|
|||
|
|
+ import java.time.Duration;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
++import java.util.Collections;
|
|||
|
|
++import java.util.HashMap;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
++import java.util.Map;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.Executors;
|
|||
|
|
+@@ -39,7 +42,10 @@ import org.apache.rocketmq.common.message.MessageClientIDSetter;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.utils.FutureUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.ProxyUtils;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.MessageQueueView;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+@@ -50,16 +56,22 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+ import org.mockito.ArgumentCaptor;
|
|||
|
|
++import org.mockito.stubbing.Answer;
|
|||
|
|
+
|
|||
|
|
+ import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
+ import static org.junit.Assert.assertEquals;
|
|||
|
|
+ import static org.junit.Assert.assertNotNull;
|
|||
|
|
++import static org.junit.Assert.assertNull;
|
|||
|
|
+ import static org.junit.Assert.assertSame;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.any;
|
|||
|
|
++import static org.mockito.ArgumentMatchers.anyList;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyLong;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyString;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.eq;
|
|||
|
|
++import static org.mockito.Mockito.doAnswer;
|
|||
|
|
+ import static org.mockito.Mockito.mock;
|
|||
|
|
++import static org.mockito.Mockito.never;
|
|||
|
|
++import static org.mockito.Mockito.verify;
|
|||
|
|
+ import static org.mockito.Mockito.when;
|
|||
|
|
+
|
|||
|
|
+ public class ConsumerProcessorTest extends BaseProcessorTest {
|
|||
|
|
+@@ -162,6 +174,109 @@ public class ConsumerProcessorTest extends BaseProcessorTest {
|
|||
|
|
+ assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testBatchAckExpireMessage() throws Throwable {
|
|||
|
|
++ String brokerName1 = "brokerName1";
|
|||
|
|
++
|
|||
|
|
++ List<ReceiptHandleMessage> receiptHandleMessageList = new ArrayList<>();
|
|||
|
|
++ for (int i = 0; i < 3; i++) {
|
|||
|
|
++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000,
|
|||
|
|
++ 0, 0, 0, i, brokerName1);
|
|||
|
|
++ ReceiptHandle expireHandle = create(expireMessage);
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId()));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ List<BatchAckResult> batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get();
|
|||
|
|
++
|
|||
|
|
++ verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong());
|
|||
|
|
++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size());
|
|||
|
|
++ for (BatchAckResult batchAckResult : batchAckResultList) {
|
|||
|
|
++ assertNull(batchAckResult.getAckResult());
|
|||
|
|
++ assertNotNull(batchAckResult.getProxyException());
|
|||
|
|
++ assertNotNull(batchAckResult.getReceiptHandleMessage());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testBatchAckMessage() throws Throwable {
|
|||
|
|
++ String brokerName1 = "brokerName1";
|
|||
|
|
++ String brokerName2 = "brokerName2";
|
|||
|
|
++ String errThrowBrokerName = "errThrowBrokerName";
|
|||
|
|
++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000,
|
|||
|
|
++ 0, 0, 0, 0, brokerName1);
|
|||
|
|
++ ReceiptHandle expireHandle = create(expireMessage);
|
|||
|
|
++
|
|||
|
|
++ List<ReceiptHandleMessage> receiptHandleMessageList = new ArrayList<>();
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId()));
|
|||
|
|
++ List<String> broker1Msg = new ArrayList<>();
|
|||
|
|
++ List<String> broker2Msg = new ArrayList<>();
|
|||
|
|
++
|
|||
|
|
++ long now = System.currentTimeMillis();
|
|||
|
|
++ int msgNum = 3;
|
|||
|
|
++ for (int i = 0; i < msgNum; i++) {
|
|||
|
|
++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now,
|
|||
|
|
++ 0, 0, 0, i + 1, brokerName1);
|
|||
|
|
++ ReceiptHandle brokerHandle = create(brokerMessage);
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId()));
|
|||
|
|
++ broker1Msg.add(brokerMessage.getMsgId());
|
|||
|
|
++ }
|
|||
|
|
++ for (int i = 0; i < msgNum; i++) {
|
|||
|
|
++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now,
|
|||
|
|
++ 0, 0, 0, i + 1, brokerName2);
|
|||
|
|
++ ReceiptHandle brokerHandle = create(brokerMessage);
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId()));
|
|||
|
|
++ broker2Msg.add(brokerMessage.getMsgId());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ // for this message, will throw exception in batchAckMessage
|
|||
|
|
++ MessageExt errThrowMessage = createMessageExt(TOPIC, "", 0, 3000, now,
|
|||
|
|
++ 0, 0, 0, 0, errThrowBrokerName);
|
|||
|
|
++ ReceiptHandle errThrowHandle = create(errThrowMessage);
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId()));
|
|||
|
|
++
|
|||
|
|
++ Collections.shuffle(receiptHandleMessageList);
|
|||
|
|
++
|
|||
|
|
++ doAnswer((Answer<CompletableFuture<AckResult>>) invocation -> {
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList = invocation.getArgument(1, List.class);
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName();
|
|||
|
|
++ if (brokerName.equals(brokerName1)) {
|
|||
|
|
++ ackResult.setStatus(AckStatus.OK);
|
|||
|
|
++ } else if (brokerName.equals(brokerName2)) {
|
|||
|
|
++ ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
++ } else {
|
|||
|
|
++ return FutureUtils.completeExceptionally(new RuntimeException());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ return CompletableFuture.completedFuture(ackResult);
|
|||
|
|
++ }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong());
|
|||
|
|
++
|
|||
|
|
++ List<BatchAckResult> batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get();
|
|||
|
|
++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size());
|
|||
|
|
++
|
|||
|
|
++ // check ackResult for each msg
|
|||
|
|
++ Map<String, BatchAckResult> msgBatchAckResult = new HashMap<>();
|
|||
|
|
++ for (BatchAckResult batchAckResult : batchAckResultList) {
|
|||
|
|
++ msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult);
|
|||
|
|
++ }
|
|||
|
|
++ for (String msgId : broker1Msg) {
|
|||
|
|
++ assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus());
|
|||
|
|
++ assertNull(msgBatchAckResult.get(msgId).getProxyException());
|
|||
|
|
++ }
|
|||
|
|
++ for (String msgId : broker2Msg) {
|
|||
|
|
++ assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus());
|
|||
|
|
++ assertNull(msgBatchAckResult.get(msgId).getProxyException());
|
|||
|
|
++ }
|
|||
|
|
++ assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException());
|
|||
|
|
++ assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode());
|
|||
|
|
++ assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult());
|
|||
|
|
++
|
|||
|
|
++ assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException());
|
|||
|
|
++ assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode());
|
|||
|
|
++ assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testChangeInvisibleTime() throws Throwable {
|
|||
|
|
+ ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000));
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
|
|||
|
|
+index 77a119a29..3f3a4ae40 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
|
|||
|
|
+@@ -220,6 +220,18 @@ public class MQClientAPIExtTest {
|
|||
|
|
+ assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testBatchAckMessageAsync() throws Exception {
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ doAnswer((Answer<Void>) mock -> {
|
|||
|
|
++ AckCallback ackCallback = mock.getArgument(2);
|
|||
|
|
++ ackCallback.onSuccess(ackResult);
|
|||
|
|
++ return null;
|
|||
|
|
++ }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any());
|
|||
|
|
++
|
|||
|
|
++ assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testChangeInvisibleTimeAsync() throws Exception {
|
|||
|
|
+ AckResult ackResult = new AckResult();
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch012-backport-enhance-rockdbconfigtojson.patch b/patch012-backport-enhance-rockdbconfigtojson.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..cf6128b24
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch012-backport-enhance-rockdbconfigtojson.patch
|
|||
|
|
@@ -0,0 +1,2920 @@
|
|||
|
|
+From fec141481496c53a0db398367006c34264662d18 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yx9o <yangx_soft@163.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 08:22:34 +0800
|
|||
|
|
+Subject: [PATCH 1/8] [ISSUE #7166] Optimize the display format of admin
|
|||
|
|
+ (#7210)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../java/org/apache/rocketmq/tools/command/MQAdminStartup.java | 2 +-
|
|||
|
|
+ .../command/acl/ClusterAclConfigVersionListSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/acl/DeleteAccessConfigSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/acl/GetAccessConfigSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/acl/UpdateAccessConfigSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/broker/BrokerConsumeStatsSubCommad.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/broker/BrokerStatusSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/broker/CommitLogSetReadAheadSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/broker/DeleteExpiredCommitLogSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/broker/GetBrokerConfigCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java | 2 +-
|
|||
|
|
+ .../broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/broker/ResetMasterFlushOffsetSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/broker/UpdateBrokerConfigSubCommand.java | 2 +-
|
|||
|
|
+ .../broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/cluster/ClusterListSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/connection/ConsumerConnectionSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/connection/ProducerConnectionSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/consumer/ConsumerStatusSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/consumer/GetConsumerConfigSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/consumer/StartMonitoringSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/consumer/UpdateSubGroupSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/container/AddBrokerSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/container/RemoveBrokerSubCommand.java | 2 +-
|
|||
|
|
+ .../command/controller/CleanControllerBrokerMetaSubCommand.java | 2 +-
|
|||
|
|
+ .../command/controller/GetControllerMetaDataSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/controller/ReElectMasterSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/export/ExportConfigsCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/export/ExportMetadataCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/export/ExportMetricsCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java | 2 +-
|
|||
|
|
+ .../apache/rocketmq/tools/command/ha/HAStatusSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/message/CheckMsgSendRTCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/message/ConsumeMessageCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/message/DumpCompactionLogCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/message/PrintMessageByQueueCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/message/PrintMessageSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/message/QueryMsgByIdSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/message/QueryMsgByKeySubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/message/QueryMsgByOffsetSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/message/QueryMsgByUniqueKeySubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/message/QueryMsgTraceByIdSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/message/SendMessageCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/namesrv/AddWritePermSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/offset/SkipAccumulationSubCommand.java | 2 +-
|
|||
|
|
+ .../apache/rocketmq/tools/command/stats/StatsAllSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/topic/AllocateMQSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/topic/TopicClusterSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/topic/TopicListSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/topic/TopicRouteSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/topic/TopicStatusSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/topic/UpdateOrderConfCommand.java | 2 +-
|
|||
|
|
+ .../tools/command/topic/UpdateStaticTopicSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java | 2 +-
|
|||
|
|
+ .../rocketmq/tools/command/topic/UpdateTopicSubCommand.java | 2 +-
|
|||
|
|
+ 59 files changed, 59 insertions(+), 59 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
+index 0c2618e91..890125ca0 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
+@@ -278,7 +278,7 @@ public class MQAdminStartup {
|
|||
|
|
+ System.out.printf("The most commonly used mqadmin commands are:%n");
|
|||
|
|
+
|
|||
|
|
+ for (SubCommand cmd : SUB_COMMANDS) {
|
|||
|
|
+- System.out.printf(" %-25s %s%n", cmd.commandName(), cmd.commandDesc());
|
|||
|
|
++ System.out.printf(" %-35s %s%n", cmd.commandName(), cmd.commandDesc());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ System.out.printf("%nSee 'mqadmin help <command>' for more information on a specific command.%n");
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java
|
|||
|
|
+index f8a00b1e0..26ed028fb 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java
|
|||
|
|
+@@ -47,7 +47,7 @@ public class ClusterAclConfigVersionListSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "List all of acl config version information in cluster";
|
|||
|
|
++ return "List all of acl config version information in cluster.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
|
|||
|
|
+index fd3a92fff..a7f3d295a 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
|
|||
|
|
+@@ -42,7 +42,7 @@ public class DeleteAccessConfigSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Delete Acl Config Account in broker";
|
|||
|
|
++ return "Delete Acl Config Account in broker.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java
|
|||
|
|
+index 25844d6a1..f1c9a1496 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java
|
|||
|
|
+@@ -49,7 +49,7 @@ public class GetAccessConfigSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "List all of acl config information in cluster";
|
|||
|
|
++ return "List all of acl config information in cluster.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java
|
|||
|
|
+index 3be40daa1..d8a06f92d 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java
|
|||
|
|
+@@ -40,7 +40,7 @@ public class UpdateAccessConfigSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Update acl config yaml file in broker";
|
|||
|
|
++ return "Update acl config yaml file in broker.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java
|
|||
|
|
+index ff662b506..9dacf1fae 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java
|
|||
|
|
+@@ -37,7 +37,7 @@ public class UpdateGlobalWhiteAddrSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Update global white address for acl Config File in broker";
|
|||
|
|
++ return "Update global white address for acl Config File in broker.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java
|
|||
|
|
+index 3f2f90673..7658a2139 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java
|
|||
|
|
+@@ -61,7 +61,7 @@ public class BrokerConsumeStatsSubCommad implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Fetch broker consume stats data";
|
|||
|
|
++ return "Fetch broker consume stats data.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java
|
|||
|
|
+index 830ff3425..ce934f547 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java
|
|||
|
|
+@@ -44,7 +44,7 @@ public class BrokerStatusSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Fetch broker runtime status data";
|
|||
|
|
++ return "Fetch broker runtime status data.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java
|
|||
|
|
+index b00c7f5f5..4fdabfdf8 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java
|
|||
|
|
+@@ -44,7 +44,7 @@ public class CommitLogSetReadAheadSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "set read ahead mode for all commitlog files";
|
|||
|
|
++ return "Set read ahead mode for all commitlog files.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java
|
|||
|
|
+index a4b2a51ad..142bb7b3c 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java
|
|||
|
|
+@@ -37,7 +37,7 @@ public class DeleteExpiredCommitLogSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Delete expired CommitLog files";
|
|||
|
|
++ return "Delete expired CommitLog files.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java
|
|||
|
|
+index 5d86c10e4..c4762a296 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java
|
|||
|
|
+@@ -45,7 +45,7 @@ public class GetBrokerConfigCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Get broker config by cluster or special broker";
|
|||
|
|
++ return "Get broker config by cluster or special broker.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java
|
|||
|
|
+index abe8fc622..1a8961e04 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java
|
|||
|
|
+@@ -38,7 +38,7 @@ public class GetBrokerEpochSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Fetch broker epoch entries";
|
|||
|
|
++ return "Fetch broker epoch entries.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java
|
|||
|
|
+index 7c54e650c..34b3ba7d3 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java
|
|||
|
|
+@@ -47,7 +47,7 @@ public class GetColdDataFlowCtrInfoSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "get cold data flow ctr info";
|
|||
|
|
++ return "Get cold data flow ctr info.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java
|
|||
|
|
+index b0477924f..f20407480 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java
|
|||
|
|
+@@ -36,7 +36,7 @@ public class RemoveColdDataFlowCtrGroupConfigSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "remove consumer from cold ctr config";
|
|||
|
|
++ return "Remove consumer from cold ctr config.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java
|
|||
|
|
+index b2ac48c84..90451b51f 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java
|
|||
|
|
+@@ -33,7 +33,7 @@ public class ResetMasterFlushOffsetSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Reset master flush offset in slave";
|
|||
|
|
++ return "Reset master flush offset in slave.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java
|
|||
|
|
+index 98abeb6ae..62816ef03 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java
|
|||
|
|
+@@ -37,7 +37,7 @@ public class UpdateBrokerConfigSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Update broker's config";
|
|||
|
|
++ return "Update broker's config.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java
|
|||
|
|
+index d06a24b57..8d1a00077 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java
|
|||
|
|
+@@ -39,7 +39,7 @@ public class UpdateColdDataFlowCtrGroupConfigSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "addOrUpdate cold data flow ctr group config";
|
|||
|
|
++ return "Add or update cold data flow ctr group config.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java
|
|||
|
|
+index 7253970bd..d755e9e5d 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java
|
|||
|
|
+@@ -48,7 +48,7 @@ public class CLusterSendMsgRTCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "List All clusters Message Send RT";
|
|||
|
|
++ return "List All clusters Message Send RT.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java
|
|||
|
|
+index a7a840a44..ede0fa5cf 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java
|
|||
|
|
+@@ -41,7 +41,7 @@ public class ClusterListSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "List cluster infos";
|
|||
|
|
++ return "List cluster infos.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java
|
|||
|
|
+index 630961e31..35f73d8a0 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java
|
|||
|
|
+@@ -39,7 +39,7 @@ public class ConsumerConnectionSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Query consumer's socket connection, client version and subscription";
|
|||
|
|
++ return "Query consumer's socket connection, client version and subscription.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java
|
|||
|
|
+index 2533982c8..bde674ab2 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java
|
|||
|
|
+@@ -36,7 +36,7 @@ public class ProducerConnectionSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Query producer's socket connection and client version";
|
|||
|
|
++ return "Query producer's socket connection and client version.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java
|
|||
|
|
+index 72b9c975e..d8f6f9aa9 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java
|
|||
|
|
+@@ -47,7 +47,7 @@ public class ConsumerStatusSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Query consumer's internal data structure";
|
|||
|
|
++ return "Query consumer's internal data structure.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java
|
|||
|
|
+index 6095e7668..4a8253a02 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java
|
|||
|
|
+@@ -43,7 +43,7 @@ public class GetConsumerConfigSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Get consumer config by subscription group name";
|
|||
|
|
++ return "Get consumer config by subscription group name.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java
|
|||
|
|
+index 2d08d0bd0..f5e140433 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java
|
|||
|
|
+@@ -34,7 +34,7 @@ public class StartMonitoringSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Start Monitoring";
|
|||
|
|
++ return "Start Monitoring.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java
|
|||
|
|
+index f87bafc93..b17da4de4 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java
|
|||
|
|
+@@ -41,7 +41,7 @@ public class UpdateSubGroupSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Update or create subscription group";
|
|||
|
|
++ return "Update or create subscription group.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java
|
|||
|
|
+index e9e5be4a5..007d42ae6 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java
|
|||
|
|
+@@ -33,7 +33,7 @@ public class AddBrokerSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Add a broker to specified container";
|
|||
|
|
++ return "Add a broker to specified container.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java
|
|||
|
|
+index 7c455f858..ab25d8ebe 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java
|
|||
|
|
+@@ -33,7 +33,7 @@ public class RemoveBrokerSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Remove a broker from specified container";
|
|||
|
|
++ return "Remove a broker from specified container.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java
|
|||
|
|
+index 856e4b426..24ed02566 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java
|
|||
|
|
+@@ -37,7 +37,7 @@ public class CleanControllerBrokerMetaSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Clean metadata of broker on controller";
|
|||
|
|
++ return "Clean metadata of broker on controller.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java
|
|||
|
|
+index 70bd7f8e9..966443127 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java
|
|||
|
|
+@@ -34,7 +34,7 @@ public class GetControllerMetaDataSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Get controller cluster's metadata";
|
|||
|
|
++ return "Get controller cluster's metadata.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java
|
|||
|
|
+index 1affe81f9..a522a903d 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java
|
|||
|
|
+@@ -37,7 +37,7 @@ public class ReElectMasterSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Re-elect the specified broker as master";
|
|||
|
|
++ return "Re-elect the specified broker as master.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java
|
|||
|
|
+index b8191296d..03613b29c 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java
|
|||
|
|
+@@ -42,7 +42,7 @@ public class ExportConfigsCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Export configs";
|
|||
|
|
++ return "Export configs.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java
|
|||
|
|
+index 1f9cf7d96..748f7b16e 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java
|
|||
|
|
+@@ -46,7 +46,7 @@ public class ExportMetadataCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Export metadata";
|
|||
|
|
++ return "Export metadata.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java
|
|||
|
|
+index a793b4b84..5d8bb37ba 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java
|
|||
|
|
+@@ -56,7 +56,7 @@ public class ExportMetricsCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Export metrics";
|
|||
|
|
++ return "Export metrics.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java
|
|||
|
|
+index 44b3ec3e1..b6231e4f9 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java
|
|||
|
|
+@@ -40,7 +40,7 @@ public class GetSyncStateSetSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Fetch syncStateSet for target brokers";
|
|||
|
|
++ return "Fetch syncStateSet for target brokers.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java
|
|||
|
|
+index b1795e046..931658a08 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java
|
|||
|
|
+@@ -41,7 +41,7 @@ public class HAStatusSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Fetch ha runtime status data";
|
|||
|
|
++ return "Fetch ha runtime status data.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java
|
|||
|
|
+index 4c6d5ffb6..b15b59d50 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java
|
|||
|
|
+@@ -40,7 +40,7 @@ public class CheckMsgSendRTCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Check message send response time";
|
|||
|
|
++ return "Check message send response time.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java
|
|||
|
|
+index 8aed59ea4..02ff53269 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java
|
|||
|
|
+@@ -70,7 +70,7 @@ public class ConsumeMessageCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Consume message";
|
|||
|
|
++ return "Consume message.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java
|
|||
|
|
+index ae6d9bdcf..eee8f3d4b 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java
|
|||
|
|
+@@ -38,7 +38,7 @@ import java.nio.file.Paths;
|
|||
|
|
+ public class DumpCompactionLogCommand implements SubCommand {
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "parse compaction log to message";
|
|||
|
|
++ return "Parse compaction log to message.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java
|
|||
|
|
+index 654560167..0418e88a7 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java
|
|||
|
|
+@@ -108,7 +108,7 @@ public class PrintMessageByQueueCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Print Message Detail";
|
|||
|
|
++ return "Print Message Detail by queueId.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java
|
|||
|
|
+index d01c36d42..bb82f5079 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java
|
|||
|
|
+@@ -62,7 +62,7 @@ public class PrintMessageSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Print Message Detail";
|
|||
|
|
++ return "Print Message Detail.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java
|
|||
|
|
+index 2880477f1..b42612150 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java
|
|||
|
|
+@@ -186,7 +186,7 @@ public class QueryMsgByIdSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Query Message by Id";
|
|||
|
|
++ return "Query Message by Id.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java
|
|||
|
|
+index ba7b00c3b..64627fd19 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java
|
|||
|
|
+@@ -36,7 +36,7 @@ public class QueryMsgByKeySubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Query Message by Key";
|
|||
|
|
++ return "Query Message by Key.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java
|
|||
|
|
+index d27313af1..14d0625fd 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java
|
|||
|
|
+@@ -39,7 +39,7 @@ public class QueryMsgByOffsetSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Query Message by offset";
|
|||
|
|
++ return "Query Message by offset.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java
|
|||
|
|
+index 1b28f8be1..b71cee901 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java
|
|||
|
|
+@@ -141,7 +141,7 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Query Message by Unique key";
|
|||
|
|
++ return "Query Message by Unique key.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java
|
|||
|
|
+index 2b982efef..2c546ec56 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java
|
|||
|
|
+@@ -65,7 +65,7 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Query a message trace";
|
|||
|
|
++ return "Query a message trace.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java
|
|||
|
|
+index 836ee192b..970da6b16 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java
|
|||
|
|
+@@ -41,7 +41,7 @@ public class SendMessageCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Send a message";
|
|||
|
|
++ return "Send a message.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java
|
|||
|
|
+index 98542d065..0b0a075bd 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java
|
|||
|
|
+@@ -34,7 +34,7 @@ public class AddWritePermSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Add write perm of broker in all name server you defined in the -n param";
|
|||
|
|
++ return "Add write perm of broker in all name server you defined in the -n param.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java
|
|||
|
|
+index 213931ed8..637dd52c8 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java
|
|||
|
|
+@@ -34,7 +34,7 @@ public class WipeWritePermSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Wipe write perm of broker in all name server you defined in the -n param";
|
|||
|
|
++ return "Wipe write perm of broker in all name server you defined in the -n param.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java
|
|||
|
|
+index 139821f9c..b22491a59 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java
|
|||
|
|
+@@ -41,7 +41,7 @@ public class SkipAccumulationSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Skip all messages that are accumulated (not consumed) currently";
|
|||
|
|
++ return "Skip all messages that are accumulated (not consumed) currently.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java
|
|||
|
|
+index 1d49bbe11..96097a93e 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java
|
|||
|
|
+@@ -144,7 +144,7 @@ public class StatsAllSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Topic and Consumer tps stats";
|
|||
|
|
++ return "Topic and Consumer tps stats.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java
|
|||
|
|
+index 3fa42f297..6a9b81eb8 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java
|
|||
|
|
+@@ -41,7 +41,7 @@ public class AllocateMQSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Allocate MQ";
|
|||
|
|
++ return "Allocate MQ.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java
|
|||
|
|
+index 1dab693d9..098f34ff0 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java
|
|||
|
|
+@@ -34,7 +34,7 @@ public class TopicClusterSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Get cluster info for topic";
|
|||
|
|
++ return "Get cluster info for topic.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java
|
|||
|
|
+index 346bac704..d9a279f80 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java
|
|||
|
|
+@@ -45,7 +45,7 @@ public class TopicListSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Fetch all topic list from name server";
|
|||
|
|
++ return "Fetch all topic list from name server.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java
|
|||
|
|
+index f2dabec4e..70949d388 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java
|
|||
|
|
+@@ -42,7 +42,7 @@ public class TopicRouteSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Examine topic route info";
|
|||
|
|
++ return "Examine topic route info.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java
|
|||
|
|
+index fdb249fab..a1619eced 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java
|
|||
|
|
+@@ -40,7 +40,7 @@ public class TopicStatusSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Examine topic Status info";
|
|||
|
|
++ return "Examine topic Status info.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java
|
|||
|
|
+index bebc646b4..3040d04c2 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java
|
|||
|
|
+@@ -36,7 +36,7 @@ public class UpdateOrderConfCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Create or update or delete order conf";
|
|||
|
|
++ return "Create or update or delete order conf.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java
|
|||
|
|
+index 85a18c654..3daeee86c 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java
|
|||
|
|
+@@ -48,7 +48,7 @@ public class UpdateStaticTopicSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Update or create static topic, which has fixed number of queues";
|
|||
|
|
++ return "Update or create static topic, which has fixed number of queues.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java
|
|||
|
|
+index aaa881538..d27cd1861 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java
|
|||
|
|
+@@ -44,7 +44,7 @@ public class UpdateTopicPermSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Update topic perm";
|
|||
|
|
++ return "Update topic perm.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java
|
|||
|
|
+index b68463396..298914175 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java
|
|||
|
|
+@@ -42,7 +42,7 @@ public class UpdateTopicSubCommand implements SubCommand {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+- return "Update or create topic";
|
|||
|
|
++ return "Update or create topic.";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 744167bd01fab6821b4d5ae1794dc845153d5156 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ziyi Tan <tanziyi0925@gmail.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 08:32:17 +0800
|
|||
|
|
+Subject: [PATCH 2/8] [ISSUE #7142] Add command `RocksDBConfigToJson` to
|
|||
|
|
+ inspect rocksdb content (#7180)
|
|||
|
|
+
|
|||
|
|
+* feat: add command `RocksDBConfigToJson` to inspect rocksdb content
|
|||
|
|
+
|
|||
|
|
+Signed-off-by: Ziy1-Tan <ajb4596984460@gmail.com>
|
|||
|
|
+
|
|||
|
|
+* refactor: fix style
|
|||
|
|
+
|
|||
|
|
+---------
|
|||
|
|
+
|
|||
|
|
+Signed-off-by: Ziy1-Tan <ajb4596984460@gmail.com>
|
|||
|
|
+Co-authored-by: Ziy1-Tan <ajb4596984460@gmail.com>
|
|||
|
|
+---
|
|||
|
|
+ .../tools/command/MQAdminStartup.java | 2 +
|
|||
|
|
+ .../metadata/RocksDBConfigToJsonCommand.java | 118 ++++++++++++++++++
|
|||
|
|
+ .../metadata/KvConfigToJsonCommandTest.java | 65 ++++++++++
|
|||
|
|
+ 3 files changed, 185 insertions(+)
|
|||
|
|
+ create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+ create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java
|
|||
|
|
+
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
+index 890125ca0..324aa1856 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
+@@ -80,6 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.message.SendMessageCommand;
|
|||
|
|
++import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand;
|
|||
|
|
+@@ -211,6 +212,7 @@ public class MQAdminStartup {
|
|||
|
|
+
|
|||
|
|
+ initCommand(new ClusterListSubCommand());
|
|||
|
|
+ initCommand(new TopicListSubCommand());
|
|||
|
|
++ initCommand(new RocksDBConfigToJsonCommand());
|
|||
|
|
+
|
|||
|
|
+ initCommand(new UpdateKvConfigCommand());
|
|||
|
|
+ initCommand(new DeleteKvConfigCommand());
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..3053f4684
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+@@ -0,0 +1,118 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++package org.apache.rocketmq.tools.command.metadata;
|
|||
|
|
++
|
|||
|
|
++import com.alibaba.fastjson.JSONObject;
|
|||
|
|
++import org.apache.commons.cli.CommandLine;
|
|||
|
|
++import org.apache.commons.cli.Option;
|
|||
|
|
++import org.apache.commons.cli.Options;
|
|||
|
|
++import org.apache.commons.lang3.StringUtils;
|
|||
|
|
++import org.apache.rocketmq.common.config.RocksDBConfigManager;
|
|||
|
|
++import org.apache.rocketmq.common.utils.DataConverter;
|
|||
|
|
++import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
++import org.apache.rocketmq.tools.command.SubCommand;
|
|||
|
|
++import org.apache.rocketmq.tools.command.SubCommandException;
|
|||
|
|
++
|
|||
|
|
++import java.io.File;
|
|||
|
|
++import java.util.HashMap;
|
|||
|
|
++import java.util.Map;
|
|||
|
|
++
|
|||
|
|
++public class RocksDBConfigToJsonCommand implements SubCommand {
|
|||
|
|
++ private static final String TOPICS_JSON_CONFIG = "topics";
|
|||
|
|
++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups";
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public String commandName() {
|
|||
|
|
++ return "rocksDBConfigToJson";
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public String commandDesc() {
|
|||
|
|
++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json";
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public Options buildCommandlineOptions(Options options) {
|
|||
|
|
++ Option pathOption = new Option("p", "path", true,
|
|||
|
|
++ "Absolute path to the metadata directory");
|
|||
|
|
++ pathOption.setRequired(true);
|
|||
|
|
++ options.addOption(pathOption);
|
|||
|
|
++
|
|||
|
|
++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " +
|
|||
|
|
++ "topics/subscriptionGroups");
|
|||
|
|
++ configTypeOption.setRequired(true);
|
|||
|
|
++ options.addOption(configTypeOption);
|
|||
|
|
++
|
|||
|
|
++ return options;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException {
|
|||
|
|
++ String path = commandLine.getOptionValue("path").trim();
|
|||
|
|
++ if (StringUtils.isEmpty(path) || !new File(path).exists()) {
|
|||
|
|
++ System.out.print("Rocksdb path is invalid.\n");
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase();
|
|||
|
|
++
|
|||
|
|
++ final long memTableFlushInterval = 60 * 60 * 1000L;
|
|||
|
|
++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval);
|
|||
|
|
++ try {
|
|||
|
|
++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
++ // for topics.json
|
|||
|
|
++ final Map<String, JSONObject> topicsJsonConfig = new HashMap<>();
|
|||
|
|
++ final Map<String, JSONObject> topicConfigTable = new HashMap<>();
|
|||
|
|
++ boolean isLoad = kvConfigManager.load(path, (key, value) -> {
|
|||
|
|
++ final String topic = new String(key, DataConverter.charset);
|
|||
|
|
++ final String topicConfig = new String(value, DataConverter.charset);
|
|||
|
|
++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig);
|
|||
|
|
++ topicConfigTable.put(topic, jsonObject);
|
|||
|
|
++ });
|
|||
|
|
++
|
|||
|
|
++ if (isLoad) {
|
|||
|
|
++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable));
|
|||
|
|
++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true);
|
|||
|
|
++ System.out.print(topicsJsonStr + "\n");
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
++ // for subscriptionGroup.json
|
|||
|
|
++ final Map<String, JSONObject> subscriptionGroupJsonConfig = new HashMap<>();
|
|||
|
|
++ final Map<String, JSONObject> subscriptionGroupTable = new HashMap<>();
|
|||
|
|
++ boolean isLoad = kvConfigManager.load(path, (key, value) -> {
|
|||
|
|
++ final String subscriptionGroup = new String(key, DataConverter.charset);
|
|||
|
|
++ final String subscriptionGroupConfig = new String(value, DataConverter.charset);
|
|||
|
|
++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig);
|
|||
|
|
++ subscriptionGroupTable.put(subscriptionGroup, jsonObject);
|
|||
|
|
++ });
|
|||
|
|
++
|
|||
|
|
++ if (isLoad) {
|
|||
|
|
++ subscriptionGroupJsonConfig.put("subscriptionGroupTable",
|
|||
|
|
++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable));
|
|||
|
|
++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true);
|
|||
|
|
++ System.out.print(subscriptionGroupJsonStr + "\n");
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ System.out.print("Config type was not recognized, configType=" + configType + "\n");
|
|||
|
|
++ } finally {
|
|||
|
|
++ kvConfigManager.stop();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..b2f66c7b0
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java
|
|||
|
|
+@@ -0,0 +1,65 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++package org.apache.rocketmq.tools.command.metadata;
|
|||
|
|
++
|
|||
|
|
++import org.apache.commons.cli.CommandLine;
|
|||
|
|
++import org.apache.commons.cli.DefaultParser;
|
|||
|
|
++import org.apache.commons.cli.Options;
|
|||
|
|
++import org.apache.rocketmq.srvutil.ServerUtil;
|
|||
|
|
++import org.apache.rocketmq.tools.command.SubCommandException;
|
|||
|
|
++import org.junit.Test;
|
|||
|
|
++
|
|||
|
|
++import java.io.File;
|
|||
|
|
++
|
|||
|
|
++import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
++
|
|||
|
|
++public class KvConfigToJsonCommandTest {
|
|||
|
|
++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/";
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testExecute() throws SubCommandException {
|
|||
|
|
++ {
|
|||
|
|
++ String[] cases = new String[]{"topics", "subscriptionGroups"};
|
|||
|
|
++ for (String c : cases) {
|
|||
|
|
++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand();
|
|||
|
|
++ Options options = ServerUtil.buildCommandlineOptions(new Options());
|
|||
|
|
++ String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c};
|
|||
|
|
++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs,
|
|||
|
|
++ cmd.buildCommandlineOptions(options), new DefaultParser());
|
|||
|
|
++ cmd.execute(commandLine, options, null);
|
|||
|
|
++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c);
|
|||
|
|
++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ // invalid cases
|
|||
|
|
++ {
|
|||
|
|
++ String[][] cases = new String[][]{
|
|||
|
|
++ {"-p " + BASE_PATH + "tmpPath", "-t topics"},
|
|||
|
|
++ {"-p ", "-t topics"},
|
|||
|
|
++ {"-p " + BASE_PATH + "topics", "-t invalid_type"}
|
|||
|
|
++ };
|
|||
|
|
++
|
|||
|
|
++ for (String[] c : cases) {
|
|||
|
|
++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand();
|
|||
|
|
++ Options options = ServerUtil.buildCommandlineOptions(new Options());
|
|||
|
|
++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c,
|
|||
|
|
++ cmd.buildCommandlineOptions(options), new DefaultParser());
|
|||
|
|
++ cmd.execute(commandLine, options, null);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From bdede35db365a49b211cdc249c68b0f60a3df46d Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: mxsm <ljbmxsm@gmail.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 08:34:56 +0800
|
|||
|
|
+Subject: [PATCH 3/8] [ISSUE #7124] Fix the typos in the code comments (#7125)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/broker/processor/ReplyMessageProcessor.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
|
|||
|
|
+index b2db356c8..d3bb048f7 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
|
|||
|
|
+@@ -234,7 +234,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor {
|
|||
|
|
+ } else {
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+ response.setRemark(null);
|
|||
|
|
+- //set to zore to avoid client decoding exception
|
|||
|
|
++ //set to zero to avoid client decoding exception
|
|||
|
|
+ responseHeader.setMsgId("0");
|
|||
|
|
+ responseHeader.setQueueId(queueIdInt);
|
|||
|
|
+ responseHeader.setQueueOffset(0L);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 9bb73b9a38548b99ac5126c40380c3c2e7fc586e Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lizhimins <707364882@qq.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 09:46:27 +0800
|
|||
|
|
+Subject: [PATCH 4/8] [#ISSUE 7222] Bug fix and refactoring of the Indexfile in
|
|||
|
|
+ tiered storage (#7224)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../tieredstore/file/TieredIndexFile.java | 38 +++++++--
|
|||
|
|
+ .../tieredstore/file/TieredIndexFileTest.java | 84 +++++--------------
|
|||
|
|
+ 2 files changed, 52 insertions(+), 70 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java
|
|||
|
|
+index 50beb01ae..eda5e0106 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java
|
|||
|
|
+@@ -16,6 +16,7 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.tieredstore.file;
|
|||
|
|
+
|
|||
|
|
++import com.google.common.annotations.VisibleForTesting;
|
|||
|
|
+ import java.io.File;
|
|||
|
|
+ import java.io.IOException;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+@@ -99,7 +100,7 @@ public class TieredIndexFile {
|
|||
|
|
+ this::doScheduleTask, 10, 10, TimeUnit.SECONDS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private void doScheduleTask() {
|
|||
|
|
++ protected void doScheduleTask() {
|
|||
|
|
+ try {
|
|||
|
|
+ curFileLock.lock();
|
|||
|
|
+ try {
|
|||
|
|
+@@ -145,6 +146,11 @@ public class TieredIndexFile {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @VisibleForTesting
|
|||
|
|
++ public MappedFile getPreMappedFile() {
|
|||
|
|
++ return preMappedFile;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ private void initFile() throws IOException {
|
|||
|
|
+ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize);
|
|||
|
|
+ initIndexFileHeader(curMappedFile);
|
|||
|
|
+@@ -156,19 +162,26 @@ public class TieredIndexFile {
|
|||
|
|
+
|
|||
|
|
+ if (isFileSealed(curMappedFile)) {
|
|||
|
|
+ if (preFileExists) {
|
|||
|
|
+- preFile.delete();
|
|||
|
|
++ if (preFile.delete()) {
|
|||
|
|
++ logger.info("Pre IndexFile deleted success", preFilepath);
|
|||
|
|
++ } else {
|
|||
|
|
++ logger.error("Pre IndexFile deleted failed", preFilepath);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ boolean rename = curMappedFile.renameTo(preFilepath);
|
|||
|
|
+ if (rename) {
|
|||
|
|
+ preMappedFile = curMappedFile;
|
|||
|
|
+ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize);
|
|||
|
|
++ initIndexFileHeader(curMappedFile);
|
|||
|
|
+ preFileExists = true;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ if (preFileExists) {
|
|||
|
|
+ synchronized (TieredIndexFile.class) {
|
|||
|
|
+ if (inflightCompactFuture.isDone()) {
|
|||
|
|
+- inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(new CompactTask(storeConfig, preMappedFile, flatFile), null);
|
|||
|
|
++ inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(
|
|||
|
|
++ new CompactTask(storeConfig, preMappedFile, flatFile), null);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+@@ -261,7 +274,8 @@ public class TieredIndexFile {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public CompletableFuture<List<Pair<Long, ByteBuffer>>> queryAsync(String topic, String key, long beginTime, long endTime) {
|
|||
|
|
++ public CompletableFuture<List<Pair<Long, ByteBuffer>>> queryAsync(String topic, String key, long beginTime,
|
|||
|
|
++ long endTime) {
|
|||
|
|
+ int hashCode = indexKeyHashMethod(buildKey(topic, key));
|
|||
|
|
+ int slotPosition = hashCode % maxHashSlotNum;
|
|||
|
|
+ List<TieredFileSegment> fileSegmentList = flatFile.getFileListByTime(beginTime, endTime);
|
|||
|
|
+@@ -355,7 +369,7 @@ public class TieredIndexFile {
|
|||
|
|
+ private final int fileMaxSize;
|
|||
|
|
+ private MappedFile originFile;
|
|||
|
|
+ private TieredFlatFile fileQueue;
|
|||
|
|
+- private final MappedFile compactFile;
|
|||
|
|
++ private MappedFile compactFile;
|
|||
|
|
+
|
|||
|
|
+ public CompactTask(TieredMessageStoreConfig storeConfig, MappedFile originFile,
|
|||
|
|
+ TieredFlatFile fileQueue) throws IOException {
|
|||
|
|
+@@ -381,6 +395,17 @@ public class TieredIndexFile {
|
|||
|
|
+ } catch (Throwable throwable) {
|
|||
|
|
+ logger.error("TieredIndexFile#compactTask: compact index file failed:", throwable);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ try {
|
|||
|
|
++ if (originFile != null) {
|
|||
|
|
++ originFile.destroy(-1);
|
|||
|
|
++ }
|
|||
|
|
++ if (compactFile != null) {
|
|||
|
|
++ compactFile.destroy(-1);
|
|||
|
|
++ }
|
|||
|
|
++ } catch (Throwable throwable) {
|
|||
|
|
++ logger.error("TieredIndexFile#compactTask: destroy index file failed:", throwable);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void compact() {
|
|||
|
|
+@@ -396,6 +421,8 @@ public class TieredIndexFile {
|
|||
|
|
+ fileQueue.commit(true);
|
|||
|
|
+ compactFile.destroy(-1);
|
|||
|
|
+ originFile.destroy(-1);
|
|||
|
|
++ compactFile = null;
|
|||
|
|
++ originFile = null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildCompactFile() {
|
|||
|
|
+@@ -414,6 +441,7 @@ public class TieredIndexFile {
|
|||
|
|
+ if (slotValue != -1) {
|
|||
|
|
+ int indexTotalSize = 0;
|
|||
|
|
+ int indexPosition = slotValue;
|
|||
|
|
++
|
|||
|
|
+ while (indexPosition >= 0 && indexPosition < maxIndexNum) {
|
|||
|
|
+ int indexOffset = INDEX_FILE_HEADER_SIZE + maxHashSlotNum * INDEX_FILE_HASH_SLOT_SIZE
|
|||
|
|
+ + indexPosition * INDEX_FILE_HASH_ORIGIN_INDEX_SIZE;
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java
|
|||
|
|
+index 7ef49578d..262d6645b 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java
|
|||
|
|
+@@ -19,9 +19,8 @@ package org.apache.rocketmq.tieredstore.file;
|
|||
|
|
+ import com.sun.jna.Platform;
|
|||
|
|
+ import java.io.IOException;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
++import java.time.Duration;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+-import java.util.concurrent.TimeUnit;
|
|||
|
|
+-import org.apache.commons.lang3.SystemUtils;
|
|||
|
|
+ import org.apache.commons.lang3.tuple.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
|
|||
|
|
+@@ -31,9 +30,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+ import org.awaitility.Awaitility;
|
|||
|
|
+ import org.junit.After;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+-import org.junit.Assume;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+-import org.junit.Ignore;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
+ public class TieredIndexFileTest {
|
|||
|
|
+@@ -45,11 +42,12 @@ public class TieredIndexFileTest {
|
|||
|
|
+ @Before
|
|||
|
|
+ public void setUp() {
|
|||
|
|
+ storeConfig = new TieredMessageStoreConfig();
|
|||
|
|
++ storeConfig.setBrokerName("IndexFileBroker");
|
|||
|
|
+ storeConfig.setStorePathRootDir(storePath);
|
|||
|
|
+- storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment");
|
|||
|
|
+- storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2);
|
|||
|
|
+- storeConfig.setTieredStoreIndexFileMaxIndexNum(3);
|
|||
|
|
+- mq = new MessageQueue("TieredIndexFileTest", storeConfig.getBrokerName(), 1);
|
|||
|
|
++ storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment");
|
|||
|
|
++ storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5);
|
|||
|
|
++ storeConfig.setTieredStoreIndexFileMaxIndexNum(20);
|
|||
|
|
++ mq = new MessageQueue("IndexFileTest", storeConfig.getBrokerName(), 1);
|
|||
|
|
+ TieredStoreUtil.getMetadataStore(storeConfig);
|
|||
|
|
+ TieredStoreExecutor.init();
|
|||
|
|
+ }
|
|||
|
|
+@@ -61,77 +59,33 @@ public class TieredIndexFileTest {
|
|||
|
|
+ TieredStoreExecutor.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- @Ignore
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testAppendAndQuery() throws IOException, ClassNotFoundException, NoSuchMethodException {
|
|||
|
|
+ if (Platform.isWindows()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- // skip this test on windows
|
|||
|
|
+- Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS);
|
|||
|
|
+-
|
|||
|
|
+ TieredFileAllocator fileQueueFactory = new TieredFileAllocator(storeConfig);
|
|||
|
|
+ TieredIndexFile indexFile = new TieredIndexFile(fileQueueFactory, storePath);
|
|||
|
|
++
|
|||
|
|
+ indexFile.append(mq, 0, "key3", 3, 300, 1000);
|
|||
|
|
+ indexFile.append(mq, 0, "key2", 2, 200, 1100);
|
|||
|
|
+ indexFile.append(mq, 0, "key1", 1, 100, 1200);
|
|||
|
|
+
|
|||
|
|
+- Awaitility.waitAtMost(5, TimeUnit.SECONDS)
|
|||
|
|
+- .until(() -> {
|
|||
|
|
+- List<Pair<Long, ByteBuffer>> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join();
|
|||
|
|
+- if (indexList.size() != 1) {
|
|||
|
|
+- return false;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- ByteBuffer indexBuffer = indexList.get(0).getValue();
|
|||
|
|
+- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 2, indexBuffer.remaining());
|
|||
|
|
+-
|
|||
|
|
+- Assert.assertEquals(1, indexBuffer.getLong(4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8 + 4));
|
|||
|
|
+-
|
|||
|
|
+- Assert.assertEquals(3, indexBuffer.getLong(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(300, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(0, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8 + 4));
|
|||
|
|
+- return true;
|
|||
|
|
+- });
|
|||
|
|
+-
|
|||
|
|
+- indexFile.append(mq, 0, "key4", 4, 400, 1300);
|
|||
|
|
+- indexFile.append(mq, 0, "key4", 4, 400, 1300);
|
|||
|
|
+- indexFile.append(mq, 0, "key4", 4, 400, 1300);
|
|||
|
|
+-
|
|||
|
|
+- Awaitility.waitAtMost(5, TimeUnit.SECONDS)
|
|||
|
|
+- .until(() -> {
|
|||
|
|
+- List<Pair<Long, ByteBuffer>> indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1300, 1300).join();
|
|||
|
|
+- if (indexList.size() != 1) {
|
|||
|
|
+- return false;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- ByteBuffer indexBuffer = indexList.get(0).getValue();
|
|||
|
|
+- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining());
|
|||
|
|
+- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4));
|
|||
|
|
+- return true;
|
|||
|
|
+- });
|
|||
|
|
+-
|
|||
|
|
+- List<Pair<Long, ByteBuffer>> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1300, 1300).join();
|
|||
|
|
++ // do not do schedule task here
|
|||
|
|
++ TieredStoreExecutor.shutdown();
|
|||
|
|
++ List<Pair<Long, ByteBuffer>> indexList =
|
|||
|
|
++ indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join();
|
|||
|
|
+ Assert.assertEquals(0, indexList.size());
|
|||
|
|
+
|
|||
|
|
+- indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1200, 1300).join();
|
|||
|
|
+- Assert.assertEquals(2, indexList.size());
|
|||
|
|
+-
|
|||
|
|
+- ByteBuffer indexBuffer = indexList.get(0).getValue();
|
|||
|
|
+- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining());
|
|||
|
|
+- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4));
|
|||
|
|
++ // do compaction once
|
|||
|
|
++ TieredStoreExecutor.init();
|
|||
|
|
++ storeConfig.setTieredStoreIndexFileRollingIdleInterval(0);
|
|||
|
|
++ indexFile.doScheduleTask();
|
|||
|
|
++ Awaitility.await().atMost(Duration.ofSeconds(10))
|
|||
|
|
++ .until(() -> !indexFile.getPreMappedFile().getFile().exists());
|
|||
|
|
+
|
|||
|
|
+- indexBuffer = indexList.get(1).getValue();
|
|||
|
|
+- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE, indexBuffer.remaining());
|
|||
|
|
+- Assert.assertEquals(2, indexBuffer.getLong(4 + 4 + 4));
|
|||
|
|
+- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8));
|
|||
|
|
+- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8 + 4));
|
|||
|
|
++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join();
|
|||
|
|
++ Assert.assertEquals(1, indexList.size());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 69c26d3d29cde7b4484ecd112ab9224f9f42bf45 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: guyinyou <36399867+guyinyou@users.noreply.github.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 10:27:52 +0800
|
|||
|
|
+Subject: [PATCH 5/8] [ISSUE #7228] Converge the use of some important
|
|||
|
|
+ variables for some class
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/store/ConsumeQueue.java | 16 ++++++------
|
|||
|
|
+ .../rocketmq/store/MappedFileQueue.java | 26 +++++++++++--------
|
|||
|
|
+ .../store/MultiPathMappedFileQueue.java | 4 +--
|
|||
|
|
+ 3 files changed, 24 insertions(+), 22 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
|
|||
|
|
+index a0b886eb0..56bee2af3 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
|
|||
|
|
+@@ -145,7 +145,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+
|
|||
|
|
+ if (offset >= 0 && size > 0) {
|
|||
|
|
+ mappedFileOffset = i + CQ_STORE_UNIT_SIZE;
|
|||
|
|
+- this.maxPhysicOffset = offset + size;
|
|||
|
|
++ this.setMaxPhysicOffset(offset + size);
|
|||
|
|
+ if (isExtAddr(tagsCode)) {
|
|||
|
|
+ maxExtAddr = tagsCode;
|
|||
|
|
+ }
|
|||
|
|
+@@ -409,7 +409,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+
|
|||
|
|
+ int logicFileSize = this.mappedFileSize;
|
|||
|
|
+
|
|||
|
|
+- this.maxPhysicOffset = phyOffset;
|
|||
|
|
++ this.setMaxPhysicOffset(phyOffset);
|
|||
|
|
+ long maxExtAddr = 1;
|
|||
|
|
+ boolean shouldDeleteFile = false;
|
|||
|
|
+ while (true) {
|
|||
|
|
+@@ -435,7 +435,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+ mappedFile.setWrotePosition(pos);
|
|||
|
|
+ mappedFile.setCommittedPosition(pos);
|
|||
|
|
+ mappedFile.setFlushedPosition(pos);
|
|||
|
|
+- this.maxPhysicOffset = offset + size;
|
|||
|
|
++ this.setMaxPhysicOffset(offset + size);
|
|||
|
|
+ // This maybe not take effect, when not every consume queue has extend file.
|
|||
|
|
+ if (isExtAddr(tagsCode)) {
|
|||
|
|
+ maxExtAddr = tagsCode;
|
|||
|
|
+@@ -453,7 +453,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+ mappedFile.setWrotePosition(pos);
|
|||
|
|
+ mappedFile.setCommittedPosition(pos);
|
|||
|
|
+ mappedFile.setFlushedPosition(pos);
|
|||
|
|
+- this.maxPhysicOffset = offset + size;
|
|||
|
|
++ this.setMaxPhysicOffset(offset + size);
|
|||
|
|
+ if (isExtAddr(tagsCode)) {
|
|||
|
|
+ maxExtAddr = tagsCode;
|
|||
|
|
+ }
|
|||
|
|
+@@ -881,8 +881,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+ private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode,
|
|||
|
|
+ final long cqOffset) {
|
|||
|
|
+
|
|||
|
|
+- if (offset + size <= this.maxPhysicOffset) {
|
|||
|
|
+- log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset);
|
|||
|
|
++ if (offset + size <= this.getMaxPhysicOffset()) {
|
|||
|
|
++ log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", this.getMaxPhysicOffset(), offset);
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -926,7 +926,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+ );
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+- this.maxPhysicOffset = offset + size;
|
|||
|
|
++ this.setMaxPhysicOffset(offset + size);
|
|||
|
|
+ return mappedFile.appendMessage(this.byteBufferIndex.array());
|
|||
|
|
+ }
|
|||
|
|
+ return false;
|
|||
|
|
+@@ -1130,7 +1130,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void destroy() {
|
|||
|
|
+- this.maxPhysicOffset = -1;
|
|||
|
|
++ this.setMaxPhysicOffset(-1);
|
|||
|
|
+ this.minLogicOffset = 0;
|
|||
|
|
+ this.mappedFileQueue.destroy();
|
|||
|
|
+ if (isExtReadEnable()) {
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
|
|||
|
|
+index 0bc70642f..32b90d14f 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
|
|||
|
|
+@@ -285,7 +285,7 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+ if (this.mappedFiles.isEmpty())
|
|||
|
|
+ return 0;
|
|||
|
|
+
|
|||
|
|
+- long committed = this.flushedWhere;
|
|||
|
|
++ long committed = this.getFlushedWhere();
|
|||
|
|
+ if (committed != 0) {
|
|||
|
|
+ MappedFile mappedFile = this.getLastMappedFile(0, false);
|
|||
|
|
+ if (mappedFile != null) {
|
|||
|
|
+@@ -442,11 +442,11 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long remainHowManyDataToCommit() {
|
|||
|
|
+- return getMaxWrotePosition() - committedWhere;
|
|||
|
|
++ return getMaxWrotePosition() - getCommittedWhere();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long remainHowManyDataToFlush() {
|
|||
|
|
+- return getMaxOffset() - flushedWhere;
|
|||
|
|
++ return getMaxOffset() - this.getFlushedWhere();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void deleteLastMappedFile() {
|
|||
|
|
+@@ -616,15 +616,15 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+
|
|||
|
|
+ public boolean flush(final int flushLeastPages) {
|
|||
|
|
+ boolean result = true;
|
|||
|
|
+- MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0);
|
|||
|
|
++ MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0);
|
|||
|
|
+ if (mappedFile != null) {
|
|||
|
|
+ long tmpTimeStamp = mappedFile.getStoreTimestamp();
|
|||
|
|
+ int offset = mappedFile.flush(flushLeastPages);
|
|||
|
|
+ long where = mappedFile.getFileFromOffset() + offset;
|
|||
|
|
+- result = where == this.flushedWhere;
|
|||
|
|
+- this.flushedWhere = where;
|
|||
|
|
++ result = where == this.getFlushedWhere();
|
|||
|
|
++ this.setFlushedWhere(where);
|
|||
|
|
+ if (0 == flushLeastPages) {
|
|||
|
|
+- this.storeTimestamp = tmpTimeStamp;
|
|||
|
|
++ this.setStoreTimestamp(tmpTimeStamp);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -633,12 +633,12 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+
|
|||
|
|
+ public synchronized boolean commit(final int commitLeastPages) {
|
|||
|
|
+ boolean result = true;
|
|||
|
|
+- MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0);
|
|||
|
|
++ MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0);
|
|||
|
|
+ if (mappedFile != null) {
|
|||
|
|
+ int offset = mappedFile.commit(commitLeastPages);
|
|||
|
|
+ long where = mappedFile.getFileFromOffset() + offset;
|
|||
|
|
+- result = where == this.committedWhere;
|
|||
|
|
+- this.committedWhere = where;
|
|||
|
|
++ result = where == this.getCommittedWhere();
|
|||
|
|
++ this.setCommittedWhere(where);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return result;
|
|||
|
|
+@@ -763,7 +763,7 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+ mf.destroy(1000 * 3);
|
|||
|
|
+ }
|
|||
|
|
+ this.mappedFiles.clear();
|
|||
|
|
+- this.flushedWhere = 0;
|
|||
|
|
++ this.setFlushedWhere(0);
|
|||
|
|
+
|
|||
|
|
+ // delete parent directory
|
|||
|
|
+ File file = new File(storePath);
|
|||
|
|
+@@ -848,6 +848,10 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+ return storeTimestamp;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public void setStoreTimestamp(long storeTimestamp) {
|
|||
|
|
++ this.storeTimestamp = storeTimestamp;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public List<MappedFile> getMappedFiles() {
|
|||
|
|
+ return mappedFiles;
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java
|
|||
|
|
+index 8f5af9438..8ff050dfe 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java
|
|||
|
|
+@@ -16,7 +16,6 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.store;
|
|||
|
|
+
|
|||
|
|
+-
|
|||
|
|
+ import java.util.Arrays;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+@@ -113,8 +112,7 @@ public class MultiPathMappedFileQueue extends MappedFileQueue {
|
|||
|
|
+ mf.destroy(1000 * 3);
|
|||
|
|
+ }
|
|||
|
|
+ this.mappedFiles.clear();
|
|||
|
|
+- this.flushedWhere = 0;
|
|||
|
|
+-
|
|||
|
|
++ this.setFlushedWhere(0);
|
|||
|
|
+
|
|||
|
|
+ Set<String> storePathSet = getPaths();
|
|||
|
|
+ storePathSet.addAll(getReadonlyPaths());
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 3884f595949462044c5cb3c236199bc1d7ad2341 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?=
|
|||
|
|
+ <shirenchuang@users.noreply.github.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 11:10:30 +0800
|
|||
|
|
+Subject: [PATCH 6/8] [ISSUE #7149] When creating and updating Topic, there
|
|||
|
|
+ will be problems with permission settings (#7151)
|
|||
|
|
+MIME-Version: 1.0
|
|||
|
|
+Content-Type: text/plain; charset=UTF-8
|
|||
|
|
+Content-Transfer-Encoding: 8bit
|
|||
|
|
+
|
|||
|
|
+* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings
|
|||
|
|
+
|
|||
|
|
+* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings
|
|||
|
|
+
|
|||
|
|
+* [issue#7249]
|
|||
|
|
+
|
|||
|
|
+---------
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: 十真 <shirenchuang.src@cainiao.com>
|
|||
|
|
+---
|
|||
|
|
+ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 3 ++-
|
|||
|
|
+ 1 file changed, 2 insertions(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+index 13f9d002b..e8f943702 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+@@ -1733,7 +1733,8 @@ public class BrokerController {
|
|||
|
|
+ new TopicConfig(topicConfig.getTopicName(),
|
|||
|
|
+ topicConfig.getReadQueueNums(),
|
|||
|
|
+ topicConfig.getWriteQueueNums(),
|
|||
|
|
+- this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag());
|
|||
|
|
++ topicConfig.getPerm()
|
|||
|
|
++ & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag());
|
|||
|
|
+ } else {
|
|||
|
|
+ registerTopicConfig = new TopicConfig(topicConfig);
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 017ad110475e8024585327b44f47e5e97aabc63b Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: echooymxq <echooy.mxq@gmail.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 11:11:42 +0800
|
|||
|
|
+Subject: [PATCH 7/8] [ISSUE #7219] Fix Concurrent modify syncStateSet and Mark
|
|||
|
|
+ synchronizing frequently when shrink. (#7220)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../broker/controller/ReplicasManager.java | 29 ++++++++++---------
|
|||
|
|
+ .../ha/autoswitch/AutoSwitchHAService.java | 21 ++++++++------
|
|||
|
|
+ 2 files changed, 28 insertions(+), 22 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
+index abae7cdb0..37c82e434 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
+@@ -542,7 +542,7 @@ public class ReplicasManager {
|
|||
|
|
+ this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId());
|
|||
|
|
+ this.tempBrokerMetadata.clear();
|
|||
|
|
+ this.brokerControllerId = this.brokerMetadata.getBrokerId();
|
|||
|
|
+- this.haService.setBrokerControllerId(this.brokerControllerId);
|
|||
|
|
++ this.haService.setLocalBrokerId(this.brokerControllerId);
|
|||
|
|
+ return true;
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ LOGGER.error("fail to create metadata file", e);
|
|||
|
|
+@@ -594,7 +594,7 @@ public class ReplicasManager {
|
|||
|
|
+ if (this.brokerMetadata.isLoaded()) {
|
|||
|
|
+ this.registerState = RegisterState.CREATE_METADATA_FILE_DONE;
|
|||
|
|
+ this.brokerControllerId = brokerMetadata.getBrokerId();
|
|||
|
|
+- this.haService.setBrokerControllerId(this.brokerControllerId);
|
|||
|
|
++ this.haService.setLocalBrokerId(this.brokerControllerId);
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ // 2. check if temp metadata exist
|
|||
|
|
+@@ -735,23 +735,26 @@ public class ReplicasManager {
|
|||
|
|
+ if (this.checkSyncStateSetTaskFuture != null) {
|
|||
|
|
+ this.checkSyncStateSetTaskFuture.cancel(false);
|
|||
|
|
+ }
|
|||
|
|
+- this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(() -> {
|
|||
|
|
+- checkSyncStateSetAndDoReport();
|
|||
|
|
+- }, 3 * 1000, this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS);
|
|||
|
|
++ this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000,
|
|||
|
|
++ this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void checkSyncStateSetAndDoReport() {
|
|||
|
|
+- final Set<Long> newSyncStateSet = this.haService.maybeShrinkSyncStateSet();
|
|||
|
|
+- newSyncStateSet.add(this.brokerControllerId);
|
|||
|
|
+- synchronized (this) {
|
|||
|
|
+- if (this.syncStateSet != null) {
|
|||
|
|
+- // Check if syncStateSet changed
|
|||
|
|
+- if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) {
|
|||
|
|
+- return;
|
|||
|
|
++ try {
|
|||
|
|
++ final Set<Long> newSyncStateSet = this.haService.maybeShrinkSyncStateSet();
|
|||
|
|
++ newSyncStateSet.add(this.brokerControllerId);
|
|||
|
|
++ synchronized (this) {
|
|||
|
|
++ if (this.syncStateSet != null) {
|
|||
|
|
++ // Check if syncStateSet changed
|
|||
|
|
++ if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++ doReportSyncStateSetChanged(newSyncStateSet);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ LOGGER.error("Check syncStateSet error", e);
|
|||
|
|
+ }
|
|||
|
|
+- doReportSyncStateSetChanged(newSyncStateSet);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void doReportSyncStateSetChanged(Set<Long> newSyncStateSet) {
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
+index 6dc734e0c..d5393fdca 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
+@@ -41,6 +41,7 @@ import java.nio.channels.SocketChannel;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
++import java.util.Iterator;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+ import java.util.Objects;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+@@ -73,7 +74,7 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
+ private EpochFileCache epochCache;
|
|||
|
|
+ private AutoSwitchHAClient haClient;
|
|||
|
|
+
|
|||
|
|
+- private Long brokerControllerId = null;
|
|||
|
|
++ private Long localBrokerId = null;
|
|||
|
|
+
|
|||
|
|
+ public AutoSwitchHAService() {
|
|||
|
|
+ }
|
|||
|
|
+@@ -287,9 +288,11 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
+
|
|||
|
|
+ // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable,
|
|||
|
|
+ // it means that the broker has not connected.
|
|||
|
|
+- for (Long slaveBrokerId : newSyncStateSet) {
|
|||
|
|
+- if (!this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) {
|
|||
|
|
+- newSyncStateSet.remove(slaveBrokerId);
|
|||
|
|
++ Iterator<Long> iterator = newSyncStateSet.iterator();
|
|||
|
|
++ while (iterator.hasNext()) {
|
|||
|
|
++ Long slaveBrokerId = iterator.next();
|
|||
|
|
++ if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) {
|
|||
|
|
++ iterator.remove();
|
|||
|
|
+ isSyncStateSetChanged = true;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+@@ -419,7 +422,7 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
+ // To avoid the syncStateSet is not consistent with connectionList.
|
|||
|
|
+ // Fix issue: https://github.com/apache/rocketmq/issues/6662
|
|||
|
|
+ for (Long syncId : currentSyncStateSet) {
|
|||
|
|
+- if (!idList.contains(syncId) && this.brokerControllerId != null && !Objects.equals(syncId, this.brokerControllerId)) {
|
|||
|
|
++ if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) {
|
|||
|
|
+ LOGGER.warn("Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.", syncId);
|
|||
|
|
+ // Without check and re-compute, return the confirmOffset's value directly.
|
|||
|
|
+ return this.defaultMessageStore.getConfirmOffsetDirectly();
|
|||
|
|
+@@ -545,12 +548,12 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
+ return this.epochCache.getAllEntries();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public Long getBrokerControllerId() {
|
|||
|
|
+- return brokerControllerId;
|
|||
|
|
++ public Long getLocalBrokerId() {
|
|||
|
|
++ return localBrokerId;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void setBrokerControllerId(Long brokerControllerId) {
|
|||
|
|
+- this.brokerControllerId = brokerControllerId;
|
|||
|
|
++ public void setLocalBrokerId(Long localBrokerId) {
|
|||
|
|
++ this.localBrokerId = localBrokerId;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ class AutoSwitchAcceptSocketService extends AcceptSocketService {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 77e8e54b37c3fc3ea0beffc1ace6f5bf20af10d9 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lk <xdkxlk@outlook.com>
|
|||
|
|
+Date: Wed, 23 Aug 2023 15:56:39 +0800
|
|||
|
|
+Subject: [PATCH 8/8] [ISSUE #7223] Support batch ack for grpc client in proxy
|
|||
|
|
+ (#7225)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../client/impl/mqclient/MQClientAPIExt.java | 26 +++
|
|||
|
|
+ .../rocketmq/proxy/config/ProxyConfig.java | 10 +
|
|||
|
|
+ .../grpc/v2/consumer/AckMessageActivity.java | 136 ++++++++---
|
|||
|
|
+ .../proxy/processor/AbstractProcessor.java | 4 +-
|
|||
|
|
+ .../proxy/processor/BatchAckResult.java | 53 +++++
|
|||
|
|
+ .../proxy/processor/ConsumerProcessor.java | 64 +++++
|
|||
|
|
+ .../processor/DefaultMessagingProcessor.java | 7 +
|
|||
|
|
+ .../proxy/processor/MessagingProcessor.java | 18 ++
|
|||
|
|
+ .../message/ClusterMessageService.java | 16 +-
|
|||
|
|
+ .../service/message/LocalMessageService.java | 58 +++++
|
|||
|
|
+ .../proxy/service/message/MessageService.java | 8 +
|
|||
|
|
+ .../service/message/ReceiptHandleMessage.java | 39 ++++
|
|||
|
|
+ .../v2/consumer/AckMessageActivityTest.java | 221 +++++++++++++++---
|
|||
|
|
+ .../proxy/processor/BaseProcessorTest.java | 18 +-
|
|||
|
|
+ .../processor/ConsumerProcessorTest.java | 115 +++++++++
|
|||
|
|
+ .../service/mqclient/MQClientAPIExtTest.java | 12 +
|
|||
|
|
+ 16 files changed, 728 insertions(+), 77 deletions(-)
|
|||
|
|
+ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java
|
|||
|
|
+ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java
|
|||
|
|
+
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
|
|||
|
|
+index fb8f8d11f..d7c8ef8d9 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
|
|||
|
|
+@@ -306,6 +306,32 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ return future;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public CompletableFuture<AckResult> batchAckMessageAsync(
|
|||
|
|
++ String brokerAddr,
|
|||
|
|
++ String topic,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ List<String> extraInfoList,
|
|||
|
|
++ long timeoutMillis
|
|||
|
|
++ ) {
|
|||
|
|
++ CompletableFuture<AckResult> future = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onSuccess(AckResult ackResult) {
|
|||
|
|
++ future.complete(ackResult);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onException(Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ }, topic, consumerGroup, extraInfoList);
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ return future;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public CompletableFuture<AckResult> changeInvisibleTimeAsync(
|
|||
|
|
+ String brokerAddr,
|
|||
|
|
+ String brokerName,
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+index 39caaa0d9..76a243919 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+@@ -250,6 +250,8 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000;
|
|||
|
|
+ private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000;
|
|||
|
|
+
|
|||
|
|
++ private boolean enableBatchAck = false;
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public void initData() {
|
|||
|
|
+ parseDelayLevel();
|
|||
|
|
+@@ -1379,4 +1381,12 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) {
|
|||
|
|
+ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isEnableBatchAck() {
|
|||
|
|
++ return enableBatchAck;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setEnableBatchAck(boolean enableBatchAck) {
|
|||
|
|
++ this.enableBatchAck = enableBatchAck;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java
|
|||
|
|
+index 9a3a77201..97c716c8f 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java
|
|||
|
|
+@@ -31,12 +31,15 @@ import org.apache.rocketmq.client.consumer.AckStatus;
|
|||
|
|
+ import org.apache.rocketmq.common.consumer.ReceiptHandle;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.MessageReceiptHandle;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
++import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;
|
|||
|
|
++import org.apache.rocketmq.proxy.processor.BatchAckResult;
|
|||
|
|
+ import org.apache.rocketmq.proxy.processor.MessagingProcessor;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+
|
|||
|
|
+ public class AckMessageActivity extends AbstractMessingActivity {
|
|||
|
|
+
|
|||
|
|
+@@ -50,60 +53,98 @@ public class AckMessageActivity extends AbstractMessingActivity {
|
|||
|
|
+
|
|||
|
|
+ try {
|
|||
|
|
+ validateTopicAndConsumerGroup(request.getTopic(), request.getGroup());
|
|||
|
|
+-
|
|||
|
|
+- CompletableFuture<AckMessageResultEntry>[] futures = new CompletableFuture[request.getEntriesCount()];
|
|||
|
|
+- for (int i = 0; i < request.getEntriesCount(); i++) {
|
|||
|
|
+- futures[i] = processAckMessage(ctx, request, request.getEntries(i));
|
|||
|
|
++ String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup());
|
|||
|
|
++ String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic());
|
|||
|
|
++ if (ConfigurationManager.getProxyConfig().isEnableBatchAck()) {
|
|||
|
|
++ future = ackMessageInBatch(ctx, group, topic, request);
|
|||
|
|
++ } else {
|
|||
|
|
++ future = ackMessageOneByOne(ctx, group, topic, request);
|
|||
|
|
+ }
|
|||
|
|
+- CompletableFuture.allOf(futures).whenComplete((val, throwable) -> {
|
|||
|
|
+- if (throwable != null) {
|
|||
|
|
+- future.completeExceptionally(throwable);
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ return future;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected CompletableFuture<AckMessageResponse> ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) {
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList = new ArrayList<>(request.getEntriesCount());
|
|||
|
|
+
|
|||
|
|
++ for (AckMessageEntry ackMessageEntry : request.getEntriesList()) {
|
|||
|
|
++ String handleString = getHandleString(ctx, group, request, ackMessageEntry);
|
|||
|
|
++ handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId()));
|
|||
|
|
++ }
|
|||
|
|
++ return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic)
|
|||
|
|
++ .thenApply(batchAckResultList -> {
|
|||
|
|
++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder();
|
|||
|
|
+ Set<Code> responseCodes = new HashSet<>();
|
|||
|
|
+- List<AckMessageResultEntry> entryList = new ArrayList<>();
|
|||
|
|
+- for (CompletableFuture<AckMessageResultEntry> entryFuture : futures) {
|
|||
|
|
+- AckMessageResultEntry entryResult = entryFuture.join();
|
|||
|
|
+- responseCodes.add(entryResult.getStatus().getCode());
|
|||
|
|
+- entryList.add(entryResult);
|
|||
|
|
+- }
|
|||
|
|
+- AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder()
|
|||
|
|
+- .addAllEntries(entryList);
|
|||
|
|
+- if (responseCodes.size() > 1) {
|
|||
|
|
+- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name()));
|
|||
|
|
+- } else if (responseCodes.size() == 1) {
|
|||
|
|
+- Code code = responseCodes.stream().findAny().get();
|
|||
|
|
+- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name()));
|
|||
|
|
+- } else {
|
|||
|
|
+- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty"));
|
|||
|
|
++ for (BatchAckResult batchAckResult : batchAckResultList) {
|
|||
|
|
++ AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult);
|
|||
|
|
++ responseBuilder.addEntries(entry);
|
|||
|
|
++ responseCodes.add(entry.getStatus().getCode());
|
|||
|
|
+ }
|
|||
|
|
+- future.complete(responseBuilder.build());
|
|||
|
|
++ setAckResponseStatus(responseBuilder, responseCodes);
|
|||
|
|
++ return responseBuilder.build();
|
|||
|
|
+ });
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) {
|
|||
|
|
++ ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage();
|
|||
|
|
++ AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder()
|
|||
|
|
++ .setMessageId(handleMessage.getMessageId())
|
|||
|
|
++ .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle());
|
|||
|
|
++ if (batchAckResult.getProxyException() != null) {
|
|||
|
|
++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException()));
|
|||
|
|
++ } else {
|
|||
|
|
++ AckResult ackResult = batchAckResult.getAckResult();
|
|||
|
|
++ if (AckStatus.OK.equals(ackResult.getStatus())) {
|
|||
|
|
++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()));
|
|||
|
|
++ } else {
|
|||
|
|
++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal"));
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+- return future;
|
|||
|
|
++ return resultBuilder.build();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- protected CompletableFuture<AckMessageResultEntry> processAckMessage(ProxyContext ctx, AckMessageRequest request,
|
|||
|
|
++ protected CompletableFuture<AckMessageResponse> ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) {
|
|||
|
|
++ CompletableFuture<AckMessageResponse> resultFuture = new CompletableFuture<>();
|
|||
|
|
++ CompletableFuture<AckMessageResultEntry>[] futures = new CompletableFuture[request.getEntriesCount()];
|
|||
|
|
++ for (int i = 0; i < request.getEntriesCount(); i++) {
|
|||
|
|
++ futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i));
|
|||
|
|
++ }
|
|||
|
|
++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> {
|
|||
|
|
++ if (throwable != null) {
|
|||
|
|
++ resultFuture.completeExceptionally(throwable);
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ Set<Code> responseCodes = new HashSet<>();
|
|||
|
|
++ List<AckMessageResultEntry> entryList = new ArrayList<>();
|
|||
|
|
++ for (CompletableFuture<AckMessageResultEntry> entryFuture : futures) {
|
|||
|
|
++ AckMessageResultEntry entryResult = entryFuture.join();
|
|||
|
|
++ responseCodes.add(entryResult.getStatus().getCode());
|
|||
|
|
++ entryList.add(entryResult);
|
|||
|
|
++ }
|
|||
|
|
++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder()
|
|||
|
|
++ .addAllEntries(entryList);
|
|||
|
|
++ setAckResponseStatus(responseBuilder, responseCodes);
|
|||
|
|
++ resultFuture.complete(responseBuilder.build());
|
|||
|
|
++ });
|
|||
|
|
++ return resultFuture;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected CompletableFuture<AckMessageResultEntry> processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request,
|
|||
|
|
+ AckMessageEntry ackMessageEntry) {
|
|||
|
|
+ CompletableFuture<AckMessageResultEntry> future = new CompletableFuture<>();
|
|||
|
|
+
|
|||
|
|
+ try {
|
|||
|
|
+- String handleString = ackMessageEntry.getReceiptHandle();
|
|||
|
|
+-
|
|||
|
|
+- String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup());
|
|||
|
|
+- MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle());
|
|||
|
|
+- if (messageReceiptHandle != null) {
|
|||
|
|
+- handleString = messageReceiptHandle.getReceiptHandleStr();
|
|||
|
|
+- }
|
|||
|
|
++ String handleString = this.getHandleString(ctx, group, request, ackMessageEntry);
|
|||
|
|
+ CompletableFuture<AckResult> ackResultFuture = this.messagingProcessor.ackMessage(
|
|||
|
|
+ ctx,
|
|||
|
|
+ ReceiptHandle.decode(handleString),
|
|||
|
|
+ ackMessageEntry.getMessageId(),
|
|||
|
|
+ group,
|
|||
|
|
+- GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()));
|
|||
|
|
++ topic
|
|||
|
|
++ );
|
|||
|
|
+ ackResultFuture.thenAccept(result -> {
|
|||
|
|
+ future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result));
|
|||
|
|
+ }).exceptionally(t -> {
|
|||
|
|
+@@ -139,4 +180,25 @@ public class AckMessageActivity extends AbstractMessingActivity {
|
|||
|
|
+ .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal"))
|
|||
|
|
+ .build();
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set<Code> responseCodes) {
|
|||
|
|
++ if (responseCodes.size() > 1) {
|
|||
|
|
++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name()));
|
|||
|
|
++ } else if (responseCodes.size() == 1) {
|
|||
|
|
++ Code code = responseCodes.stream().findAny().get();
|
|||
|
|
++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name()));
|
|||
|
|
++ } else {
|
|||
|
|
++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty"));
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) {
|
|||
|
|
++ String handleString = ackMessageEntry.getReceiptHandle();
|
|||
|
|
++
|
|||
|
|
++ MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle());
|
|||
|
|
++ if (messageReceiptHandle != null) {
|
|||
|
|
++ handleString = messageReceiptHandle.getReceiptHandleStr();
|
|||
|
|
++ }
|
|||
|
|
++ return handleString;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java
|
|||
|
|
+index b61c3df9e..c63212c23 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java
|
|||
|
|
+@@ -27,6 +27,8 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown {
|
|||
|
|
+ protected MessagingProcessor messagingProcessor;
|
|||
|
|
+ protected ServiceManager serviceManager;
|
|||
|
|
+
|
|||
|
|
++ protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired");
|
|||
|
|
++
|
|||
|
|
+ public AbstractProcessor(MessagingProcessor messagingProcessor,
|
|||
|
|
+ ServiceManager serviceManager) {
|
|||
|
|
+ this.messagingProcessor = messagingProcessor;
|
|||
|
|
+@@ -35,7 +37,7 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown {
|
|||
|
|
+
|
|||
|
|
+ protected void validateReceiptHandle(ReceiptHandle handle) {
|
|||
|
|
+ if (handle.isExpired()) {
|
|||
|
|
+- throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired");
|
|||
|
|
++ throw EXPIRED_HANDLE_PROXY_EXCEPTION;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..dfb9c9b9e
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java
|
|||
|
|
+@@ -0,0 +1,53 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++
|
|||
|
|
++package org.apache.rocketmq.proxy.processor;
|
|||
|
|
++
|
|||
|
|
++import org.apache.rocketmq.client.consumer.AckResult;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.ProxyException;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
++
|
|||
|
|
++public class BatchAckResult {
|
|||
|
|
++
|
|||
|
|
++ private final ReceiptHandleMessage receiptHandleMessage;
|
|||
|
|
++ private AckResult ackResult;
|
|||
|
|
++ private ProxyException proxyException;
|
|||
|
|
++
|
|||
|
|
++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage,
|
|||
|
|
++ AckResult ackResult) {
|
|||
|
|
++ this.receiptHandleMessage = receiptHandleMessage;
|
|||
|
|
++ this.ackResult = ackResult;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage,
|
|||
|
|
++ ProxyException proxyException) {
|
|||
|
|
++ this.receiptHandleMessage = receiptHandleMessage;
|
|||
|
|
++ this.proxyException = proxyException;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public ReceiptHandleMessage getReceiptHandleMessage() {
|
|||
|
|
++ return receiptHandleMessage;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public AckResult getAckResult() {
|
|||
|
|
++ return ackResult;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public ProxyException getProxyException() {
|
|||
|
|
++ return proxyException;
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java
|
|||
|
|
+index 656a6339d..f3522b374 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java
|
|||
|
|
+@@ -48,6 +48,7 @@ import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.FutureUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.ProxyUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.ServiceManager;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
|
|||
|
|
+@@ -241,6 +242,69 @@ public class ConsumerProcessor extends AbstractProcessor {
|
|||
|
|
+ return FutureUtils.addExecutor(future, this.executor);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public CompletableFuture<List<BatchAckResult>> batchAckMessage(
|
|||
|
|
++ ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ String topic,
|
|||
|
|
++ long timeoutMillis
|
|||
|
|
++ ) {
|
|||
|
|
++ CompletableFuture<List<BatchAckResult>> future = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ List<BatchAckResult> batchAckResultList = new ArrayList<>(handleMessageList.size());
|
|||
|
|
++ Map<String, List<ReceiptHandleMessage>> brokerHandleListMap = new HashMap<>();
|
|||
|
|
++
|
|||
|
|
++ for (ReceiptHandleMessage handleMessage : handleMessageList) {
|
|||
|
|
++ if (handleMessage.getReceiptHandle().isExpired()) {
|
|||
|
|
++ batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION));
|
|||
|
|
++ continue;
|
|||
|
|
++ }
|
|||
|
|
++ List<ReceiptHandleMessage> brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>());
|
|||
|
|
++ brokerHandleList.add(handleMessage);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ if (brokerHandleListMap.isEmpty()) {
|
|||
|
|
++ return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor);
|
|||
|
|
++ }
|
|||
|
|
++ Set<Map.Entry<String, List<ReceiptHandleMessage>>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet();
|
|||
|
|
++ CompletableFuture<List<BatchAckResult>>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()];
|
|||
|
|
++ int futureIndex = 0;
|
|||
|
|
++ for (Map.Entry<String, List<ReceiptHandleMessage>> entry : brokerHandleListMapEntrySet) {
|
|||
|
|
++ futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis);
|
|||
|
|
++ }
|
|||
|
|
++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> {
|
|||
|
|
++ if (throwable != null) {
|
|||
|
|
++ future.completeExceptionally(throwable);
|
|||
|
|
++ }
|
|||
|
|
++ for (CompletableFuture<List<BatchAckResult>> resultFuture : futures) {
|
|||
|
|
++ batchAckResultList.addAll(resultFuture.join());
|
|||
|
|
++ }
|
|||
|
|
++ future.complete(batchAckResultList);
|
|||
|
|
++ });
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ return FutureUtils.addExecutor(future, this.executor);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected CompletableFuture<List<BatchAckResult>> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List<ReceiptHandleMessage> handleMessageList, long timeoutMillis) {
|
|||
|
|
++ return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis)
|
|||
|
|
++ .thenApply(result -> {
|
|||
|
|
++ List<BatchAckResult> results = new ArrayList<>();
|
|||
|
|
++ for (ReceiptHandleMessage handleMessage : handleMessageList) {
|
|||
|
|
++ results.add(new BatchAckResult(handleMessage, result));
|
|||
|
|
++ }
|
|||
|
|
++ return results;
|
|||
|
|
++ })
|
|||
|
|
++ .exceptionally(throwable -> {
|
|||
|
|
++ List<BatchAckResult> results = new ArrayList<>();
|
|||
|
|
++ for (ReceiptHandleMessage handleMessage : handleMessageList) {
|
|||
|
|
++ results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable)));
|
|||
|
|
++ }
|
|||
|
|
++ return results;
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public CompletableFuture<AckResult> changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle,
|
|||
|
|
+ String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) {
|
|||
|
|
+ CompletableFuture<AckResult> future = new CompletableFuture<>();
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
|
|||
|
|
+index 188cb7b9b..ba150051b 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
|
|||
|
|
+@@ -46,6 +46,7 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ProxyConfig;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.ServiceManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.ServiceManagerFactory;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.metadata.MetadataService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;
|
|||
|
|
+@@ -183,6 +184,12 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen
|
|||
|
|
+ return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public CompletableFuture<List<BatchAckResult>> batchAckMessage(ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList, String consumerGroup, String topic, long timeoutMillis) {
|
|||
|
|
++ return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<AckResult> changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId,
|
|||
|
|
+ String groupName, String topicName, long invisibleTime, long timeoutMillis) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
|
|||
|
|
+index d86be0bd8..2ae7418ba 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
|
|||
|
|
+@@ -37,6 +37,7 @@ import org.apache.rocketmq.proxy.common.Address;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.MessageReceiptHandle;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.StartAndShutdown;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.metadata.MetadataService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;
|
|||
|
|
+@@ -155,6 +156,23 @@ public interface MessagingProcessor extends StartAndShutdown {
|
|||
|
|
+ long timeoutMillis
|
|||
|
|
+ );
|
|||
|
|
+
|
|||
|
|
++ default CompletableFuture<List<BatchAckResult>> batchAckMessage(
|
|||
|
|
++ ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ String topic
|
|||
|
|
++ ) {
|
|||
|
|
++ return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ CompletableFuture<List<BatchAckResult>> batchAckMessage(
|
|||
|
|
++ ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ String topic,
|
|||
|
|
++ long timeoutMillis
|
|||
|
|
++ );
|
|||
|
|
++
|
|||
|
|
+ default CompletableFuture<AckResult> changeInvisibleTime(
|
|||
|
|
+ ProxyContext ctx,
|
|||
|
|
+ ReceiptHandle handle,
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
|
|||
|
|
+index 9f163f1b9..70b72deae 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
|
|||
|
|
+@@ -20,9 +20,11 @@ import com.google.common.collect.Lists;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
++import java.util.stream.Collectors;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckResult;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.PopResult;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.PullResult;
|
|||
|
|
++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.SendResult;
|
|||
|
|
+ import org.apache.rocketmq.common.consumer.ReceiptHandle;
|
|||
|
|
+ import org.apache.rocketmq.common.message.Message;
|
|||
|
|
+@@ -31,7 +33,6 @@ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyException;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.FutureUtils;
|
|||
|
|
+-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.TopicRouteService;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+@@ -137,6 +138,19 @@ public class ClusterMessageService implements MessageService {
|
|||
|
|
+ );
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public CompletableFuture<AckResult> batchAckMessage(ProxyContext ctx, List<ReceiptHandleMessage> handleList, String consumerGroup,
|
|||
|
|
++ String topic, long timeoutMillis) {
|
|||
|
|
++ List<String> extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList());
|
|||
|
|
++ return this.mqClientAPIFactory.getClient().batchAckMessageAsync(
|
|||
|
|
++ this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()),
|
|||
|
|
++ topic,
|
|||
|
|
++ consumerGroup,
|
|||
|
|
++ extraInfoList,
|
|||
|
|
++ timeoutMillis
|
|||
|
|
++ );
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<PullResult> pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,
|
|||
|
|
+ PullMessageRequestHeader requestHeader, long timeoutMillis) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
|
|||
|
|
+index eb2c4d9ee..ca7dcc9eb 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
|
|||
|
|
+@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.service.message;
|
|||
|
|
+ import io.netty.channel.ChannelHandlerContext;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
++import java.util.BitSet;
|
|||
|
|
+ import java.util.Collections;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+@@ -54,6 +55,8 @@ import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RequestCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.ResponseCode;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.body.BatchAck;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;
|
|||
|
|
+@@ -364,6 +367,61 @@ public class LocalMessageService implements MessageService {
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public CompletableFuture<AckResult> batchAckMessage(ProxyContext ctx, List<ReceiptHandleMessage> handleList,
|
|||
|
|
++ String consumerGroup, String topic, long timeoutMillis) {
|
|||
|
|
++ SimpleChannel channel = channelManager.createChannel(ctx);
|
|||
|
|
++ ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext();
|
|||
|
|
++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);
|
|||
|
|
++
|
|||
|
|
++ Map<String, BatchAck> batchAckMap = new HashMap<>();
|
|||
|
|
++ for (ReceiptHandleMessage receiptHandleMessage : handleList) {
|
|||
|
|
++ String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle();
|
|||
|
|
++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo);
|
|||
|
|
++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" +
|
|||
|
|
++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" +
|
|||
|
|
++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" +
|
|||
|
|
++ ExtraInfoUtil.getPopTime(extraInfoData);
|
|||
|
|
++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> {
|
|||
|
|
++ BatchAck newBatchAck = new BatchAck();
|
|||
|
|
++ newBatchAck.setConsumerGroup(consumerGroup);
|
|||
|
|
++ newBatchAck.setTopic(topic);
|
|||
|
|
++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData));
|
|||
|
|
++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData));
|
|||
|
|
++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData));
|
|||
|
|
++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData));
|
|||
|
|
++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData));
|
|||
|
|
++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData));
|
|||
|
|
++ newBatchAck.setBitSet(new BitSet());
|
|||
|
|
++ return newBatchAck;
|
|||
|
|
++ });
|
|||
|
|
++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData)));
|
|||
|
|
++ }
|
|||
|
|
++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody();
|
|||
|
|
++ requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName());
|
|||
|
|
++ requestBody.setAcks(new ArrayList<>(batchAckMap.values()));
|
|||
|
|
++
|
|||
|
|
++ command.setBody(requestBody.encode());
|
|||
|
|
++ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ RemotingCommand response = brokerController.getAckMessageProcessor()
|
|||
|
|
++ .processRequest(channelHandlerContext, command);
|
|||
|
|
++ future.complete(response);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ log.error("Fail to process batchAckMessage command", e);
|
|||
|
|
++ future.completeExceptionally(e);
|
|||
|
|
++ }
|
|||
|
|
++ return future.thenApply(r -> {
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ if (ResponseCode.SUCCESS == r.getCode()) {
|
|||
|
|
++ ackResult.setStatus(AckStatus.OK);
|
|||
|
|
++ } else {
|
|||
|
|
++ ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
++ }
|
|||
|
|
++ return ackResult;
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<PullResult> pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,
|
|||
|
|
+ PullMessageRequestHeader requestHeader, long timeoutMillis) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
|
|||
|
|
+index 15da17154..58a835adb 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
|
|||
|
|
+@@ -91,6 +91,14 @@ public interface MessageService {
|
|||
|
|
+ long timeoutMillis
|
|||
|
|
+ );
|
|||
|
|
+
|
|||
|
|
++ CompletableFuture<AckResult> batchAckMessage(
|
|||
|
|
++ ProxyContext ctx,
|
|||
|
|
++ List<ReceiptHandleMessage> handleList,
|
|||
|
|
++ String consumerGroup,
|
|||
|
|
++ String topic,
|
|||
|
|
++ long timeoutMillis
|
|||
|
|
++ );
|
|||
|
|
++
|
|||
|
|
+ CompletableFuture<PullResult> pullMessage(
|
|||
|
|
+ ProxyContext ctx,
|
|||
|
|
+ AddressableMessageQueue messageQueue,
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..ae63fed49
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java
|
|||
|
|
+@@ -0,0 +1,39 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++
|
|||
|
|
++package org.apache.rocketmq.proxy.service.message;
|
|||
|
|
++
|
|||
|
|
++import org.apache.rocketmq.common.consumer.ReceiptHandle;
|
|||
|
|
++
|
|||
|
|
++public class ReceiptHandleMessage {
|
|||
|
|
++
|
|||
|
|
++ private final ReceiptHandle receiptHandle;
|
|||
|
|
++ private final String messageId;
|
|||
|
|
++
|
|||
|
|
++ public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) {
|
|||
|
|
++ this.receiptHandle = receiptHandle;
|
|||
|
|
++ this.messageId = messageId;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public ReceiptHandle getReceiptHandle() {
|
|||
|
|
++ return receiptHandle;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public String getMessageId() {
|
|||
|
|
++ return messageId;
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java
|
|||
|
|
+index 49fdfc6a8..3c4746105 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java
|
|||
|
|
+@@ -20,21 +20,32 @@ package org.apache.rocketmq.proxy.grpc.v2.consumer;
|
|||
|
|
+ import apache.rocketmq.v2.AckMessageEntry;
|
|||
|
|
+ import apache.rocketmq.v2.AckMessageRequest;
|
|||
|
|
+ import apache.rocketmq.v2.AckMessageResponse;
|
|||
|
|
++import apache.rocketmq.v2.AckMessageResultEntry;
|
|||
|
|
+ import apache.rocketmq.v2.Code;
|
|||
|
|
+ import apache.rocketmq.v2.Resource;
|
|||
|
|
++import java.util.ArrayList;
|
|||
|
|
++import java.util.HashMap;
|
|||
|
|
++import java.util.List;
|
|||
|
|
++import java.util.Map;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckResult;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckStatus;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyException;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
|
|||
|
|
++import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;
|
|||
|
|
++import org.apache.rocketmq.proxy.processor.BatchAckResult;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
++import org.mockito.stubbing.Answer;
|
|||
|
|
+
|
|||
|
|
+ import static org.junit.Assert.assertEquals;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.any;
|
|||
|
|
++import static org.mockito.ArgumentMatchers.anyList;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyString;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.eq;
|
|||
|
|
++import static org.mockito.Mockito.doAnswer;
|
|||
|
|
+ import static org.mockito.Mockito.when;
|
|||
|
|
+
|
|||
|
|
+ public class AckMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+@@ -52,43 +63,197 @@ public class AckMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testAckMessage() throws Throwable {
|
|||
|
|
+- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString()))
|
|||
|
|
++ ConfigurationManager.getProxyConfig().setEnableBatchAck(false);
|
|||
|
|
++
|
|||
|
|
++ String msg1 = "msg1";
|
|||
|
|
++ String msg2 = "msg2";
|
|||
|
|
++ String msg3 = "msg3";
|
|||
|
|
++
|
|||
|
|
++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString()))
|
|||
|
|
+ .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"));
|
|||
|
|
+
|
|||
|
|
+ AckResult msg2AckResult = new AckResult();
|
|||
|
|
+ msg2AckResult.setStatus(AckStatus.OK);
|
|||
|
|
+- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString()))
|
|||
|
|
++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString()))
|
|||
|
|
+ .thenReturn(CompletableFuture.completedFuture(msg2AckResult));
|
|||
|
|
+
|
|||
|
|
+ AckResult msg3AckResult = new AckResult();
|
|||
|
|
+ msg3AckResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
+- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString()))
|
|||
|
|
++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString()))
|
|||
|
|
+ .thenReturn(CompletableFuture.completedFuture(msg3AckResult));
|
|||
|
|
+
|
|||
|
|
+- AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
+- createContext(),
|
|||
|
|
+- AckMessageRequest.newBuilder()
|
|||
|
|
+- .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
+- .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
+- .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
+- .setMessageId("msg1")
|
|||
|
|
+- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
+- .build())
|
|||
|
|
+- .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
+- .setMessageId("msg2")
|
|||
|
|
+- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
+- .build())
|
|||
|
|
+- .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
+- .setMessageId("msg3")
|
|||
|
|
+- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
+- .build())
|
|||
|
|
+- .build()
|
|||
|
|
+- ).get();
|
|||
|
|
+-
|
|||
|
|
+- assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode());
|
|||
|
|
+- assertEquals(3, response.getEntriesCount());
|
|||
|
|
+- assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode());
|
|||
|
|
+- assertEquals(Code.OK, response.getEntries(1).getStatus().getCode());
|
|||
|
|
+- assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode());
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg1)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg2)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.OK, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg3)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg1)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg2)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(msg3)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++
|
|||
|
|
++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode());
|
|||
|
|
++ assertEquals(3, response.getEntriesCount());
|
|||
|
|
++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode());
|
|||
|
|
++ assertEquals(Code.OK, response.getEntries(1).getStatus().getCode());
|
|||
|
|
++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testAckMessageInBatch() throws Throwable {
|
|||
|
|
++ ConfigurationManager.getProxyConfig().setEnableBatchAck(true);
|
|||
|
|
++
|
|||
|
|
++ String successMessageId = "msg1";
|
|||
|
|
++ String notOkMessageId = "msg2";
|
|||
|
|
++ String exceptionMessageId = "msg3";
|
|||
|
|
++
|
|||
|
|
++ doAnswer((Answer<CompletableFuture<List<BatchAckResult>>>) invocation -> {
|
|||
|
|
++ List<ReceiptHandleMessage> receiptHandleMessageList = invocation.getArgument(1, List.class);
|
|||
|
|
++ List<BatchAckResult> batchAckResultList = new ArrayList<>();
|
|||
|
|
++ for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) {
|
|||
|
|
++ BatchAckResult batchAckResult;
|
|||
|
|
++ if (receiptHandleMessage.getMessageId().equals(successMessageId)) {
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ ackResult.setStatus(AckStatus.OK);
|
|||
|
|
++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult);
|
|||
|
|
++ } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) {
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult);
|
|||
|
|
++ } else {
|
|||
|
|
++ batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, ""));
|
|||
|
|
++ }
|
|||
|
|
++ batchAckResultList.add(batchAckResult);
|
|||
|
|
++ }
|
|||
|
|
++ return CompletableFuture.completedFuture(batchAckResultList);
|
|||
|
|
++ }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString());
|
|||
|
|
++
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(successMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.OK, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(notOkMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(exceptionMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ {
|
|||
|
|
++ AckMessageResponse response = this.ackMessageActivity.ackMessage(
|
|||
|
|
++ createContext(),
|
|||
|
|
++ AckMessageRequest.newBuilder()
|
|||
|
|
++ .setTopic(Resource.newBuilder().setName(TOPIC).build())
|
|||
|
|
++ .setGroup(Resource.newBuilder().setName(GROUP).build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(successMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(notOkMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .addEntries(AckMessageEntry.newBuilder()
|
|||
|
|
++ .setMessageId(exceptionMessageId)
|
|||
|
|
++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))
|
|||
|
|
++ .build())
|
|||
|
|
++ .build()
|
|||
|
|
++ ).get();
|
|||
|
|
++
|
|||
|
|
++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode());
|
|||
|
|
++ assertEquals(3, response.getEntriesCount());
|
|||
|
|
++ Map<String, Code> msgCode = new HashMap<>();
|
|||
|
|
++ for (AckMessageResultEntry entry : response.getEntriesList()) {
|
|||
|
|
++ msgCode.put(entry.getMessageId(), entry.getStatus().getCode());
|
|||
|
|
++ }
|
|||
|
|
++ assertEquals(Code.OK, msgCode.get(successMessageId));
|
|||
|
|
++ assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId));
|
|||
|
|
++ assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId));
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+\ No newline at end of file
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java
|
|||
|
|
+index 5c1ea9627..072630e39 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java
|
|||
|
|
+@@ -66,14 +66,6 @@ public class BaseProcessorTest extends InitConfigTest {
|
|||
|
|
+ protected ProxyRelayService proxyRelayService;
|
|||
|
|
+ @Mock
|
|||
|
|
+ protected MetadataService metadataService;
|
|||
|
|
+- @Mock
|
|||
|
|
+- protected ProducerProcessor producerProcessor;
|
|||
|
|
+- @Mock
|
|||
|
|
+- protected ConsumerProcessor consumerProcessor;
|
|||
|
|
+- @Mock
|
|||
|
|
+- protected TransactionProcessor transactionProcessor;
|
|||
|
|
+- @Mock
|
|||
|
|
+- protected ClientProcessor clientProcessor;
|
|||
|
|
+
|
|||
|
|
+ public void before() throws Throwable {
|
|||
|
|
+ super.before();
|
|||
|
|
+@@ -92,6 +84,13 @@ public class BaseProcessorTest extends InitConfigTest {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) {
|
|||
|
|
++ return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(),
|
|||
|
|
++ RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE),
|
|||
|
|
++ RANDOM.nextInt(Integer.MAX_VALUE), "mockBroker");
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime,
|
|||
|
|
++ long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) {
|
|||
|
|
+ MessageExt messageExt = new MessageExt();
|
|||
|
|
+ messageExt.setTopic(topic);
|
|||
|
|
+ messageExt.setTags(tags);
|
|||
|
|
+@@ -100,8 +99,7 @@ public class BaseProcessorTest extends InitConfigTest {
|
|||
|
|
+ messageExt.setMsgId(MessageClientIDSetter.createUniqID());
|
|||
|
|
+ messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE));
|
|||
|
|
+ MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK,
|
|||
|
|
+- ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime,
|
|||
|
|
+- RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE)));
|
|||
|
|
++ ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset));
|
|||
|
|
+ return messageExt;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java
|
|||
|
|
+index 717e86fc0..db268a06e 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java
|
|||
|
|
+@@ -20,8 +20,11 @@ package org.apache.rocketmq.proxy.processor;
|
|||
|
|
+ import com.google.common.collect.Sets;
|
|||
|
|
+ import java.time.Duration;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
++import java.util.Collections;
|
|||
|
|
++import java.util.HashMap;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
++import java.util.Map;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.Executors;
|
|||
|
|
+@@ -39,7 +42,10 @@ import org.apache.rocketmq.common.message.MessageClientIDSetter;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.utils.FutureUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.ProxyUtils;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.MessageQueueView;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+@@ -50,16 +56,22 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+ import org.mockito.ArgumentCaptor;
|
|||
|
|
++import org.mockito.stubbing.Answer;
|
|||
|
|
+
|
|||
|
|
+ import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
+ import static org.junit.Assert.assertEquals;
|
|||
|
|
+ import static org.junit.Assert.assertNotNull;
|
|||
|
|
++import static org.junit.Assert.assertNull;
|
|||
|
|
+ import static org.junit.Assert.assertSame;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.any;
|
|||
|
|
++import static org.mockito.ArgumentMatchers.anyList;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyLong;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyString;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.eq;
|
|||
|
|
++import static org.mockito.Mockito.doAnswer;
|
|||
|
|
+ import static org.mockito.Mockito.mock;
|
|||
|
|
++import static org.mockito.Mockito.never;
|
|||
|
|
++import static org.mockito.Mockito.verify;
|
|||
|
|
+ import static org.mockito.Mockito.when;
|
|||
|
|
+
|
|||
|
|
+ public class ConsumerProcessorTest extends BaseProcessorTest {
|
|||
|
|
+@@ -162,6 +174,109 @@ public class ConsumerProcessorTest extends BaseProcessorTest {
|
|||
|
|
+ assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testBatchAckExpireMessage() throws Throwable {
|
|||
|
|
++ String brokerName1 = "brokerName1";
|
|||
|
|
++
|
|||
|
|
++ List<ReceiptHandleMessage> receiptHandleMessageList = new ArrayList<>();
|
|||
|
|
++ for (int i = 0; i < 3; i++) {
|
|||
|
|
++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000,
|
|||
|
|
++ 0, 0, 0, i, brokerName1);
|
|||
|
|
++ ReceiptHandle expireHandle = create(expireMessage);
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId()));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ List<BatchAckResult> batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get();
|
|||
|
|
++
|
|||
|
|
++ verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong());
|
|||
|
|
++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size());
|
|||
|
|
++ for (BatchAckResult batchAckResult : batchAckResultList) {
|
|||
|
|
++ assertNull(batchAckResult.getAckResult());
|
|||
|
|
++ assertNotNull(batchAckResult.getProxyException());
|
|||
|
|
++ assertNotNull(batchAckResult.getReceiptHandleMessage());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testBatchAckMessage() throws Throwable {
|
|||
|
|
++ String brokerName1 = "brokerName1";
|
|||
|
|
++ String brokerName2 = "brokerName2";
|
|||
|
|
++ String errThrowBrokerName = "errThrowBrokerName";
|
|||
|
|
++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000,
|
|||
|
|
++ 0, 0, 0, 0, brokerName1);
|
|||
|
|
++ ReceiptHandle expireHandle = create(expireMessage);
|
|||
|
|
++
|
|||
|
|
++ List<ReceiptHandleMessage> receiptHandleMessageList = new ArrayList<>();
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId()));
|
|||
|
|
++ List<String> broker1Msg = new ArrayList<>();
|
|||
|
|
++ List<String> broker2Msg = new ArrayList<>();
|
|||
|
|
++
|
|||
|
|
++ long now = System.currentTimeMillis();
|
|||
|
|
++ int msgNum = 3;
|
|||
|
|
++ for (int i = 0; i < msgNum; i++) {
|
|||
|
|
++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now,
|
|||
|
|
++ 0, 0, 0, i + 1, brokerName1);
|
|||
|
|
++ ReceiptHandle brokerHandle = create(brokerMessage);
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId()));
|
|||
|
|
++ broker1Msg.add(brokerMessage.getMsgId());
|
|||
|
|
++ }
|
|||
|
|
++ for (int i = 0; i < msgNum; i++) {
|
|||
|
|
++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now,
|
|||
|
|
++ 0, 0, 0, i + 1, brokerName2);
|
|||
|
|
++ ReceiptHandle brokerHandle = create(brokerMessage);
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId()));
|
|||
|
|
++ broker2Msg.add(brokerMessage.getMsgId());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ // for this message, will throw exception in batchAckMessage
|
|||
|
|
++ MessageExt errThrowMessage = createMessageExt(TOPIC, "", 0, 3000, now,
|
|||
|
|
++ 0, 0, 0, 0, errThrowBrokerName);
|
|||
|
|
++ ReceiptHandle errThrowHandle = create(errThrowMessage);
|
|||
|
|
++ receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId()));
|
|||
|
|
++
|
|||
|
|
++ Collections.shuffle(receiptHandleMessageList);
|
|||
|
|
++
|
|||
|
|
++ doAnswer((Answer<CompletableFuture<AckResult>>) invocation -> {
|
|||
|
|
++ List<ReceiptHandleMessage> handleMessageList = invocation.getArgument(1, List.class);
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName();
|
|||
|
|
++ if (brokerName.equals(brokerName1)) {
|
|||
|
|
++ ackResult.setStatus(AckStatus.OK);
|
|||
|
|
++ } else if (brokerName.equals(brokerName2)) {
|
|||
|
|
++ ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
++ } else {
|
|||
|
|
++ return FutureUtils.completeExceptionally(new RuntimeException());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ return CompletableFuture.completedFuture(ackResult);
|
|||
|
|
++ }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong());
|
|||
|
|
++
|
|||
|
|
++ List<BatchAckResult> batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get();
|
|||
|
|
++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size());
|
|||
|
|
++
|
|||
|
|
++ // check ackResult for each msg
|
|||
|
|
++ Map<String, BatchAckResult> msgBatchAckResult = new HashMap<>();
|
|||
|
|
++ for (BatchAckResult batchAckResult : batchAckResultList) {
|
|||
|
|
++ msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult);
|
|||
|
|
++ }
|
|||
|
|
++ for (String msgId : broker1Msg) {
|
|||
|
|
++ assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus());
|
|||
|
|
++ assertNull(msgBatchAckResult.get(msgId).getProxyException());
|
|||
|
|
++ }
|
|||
|
|
++ for (String msgId : broker2Msg) {
|
|||
|
|
++ assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus());
|
|||
|
|
++ assertNull(msgBatchAckResult.get(msgId).getProxyException());
|
|||
|
|
++ }
|
|||
|
|
++ assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException());
|
|||
|
|
++ assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode());
|
|||
|
|
++ assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult());
|
|||
|
|
++
|
|||
|
|
++ assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException());
|
|||
|
|
++ assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode());
|
|||
|
|
++ assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testChangeInvisibleTime() throws Throwable {
|
|||
|
|
+ ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000));
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
|
|||
|
|
+index 77a119a29..3f3a4ae40 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
|
|||
|
|
+@@ -220,6 +220,18 @@ public class MQClientAPIExtTest {
|
|||
|
|
+ assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testBatchAckMessageAsync() throws Exception {
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ doAnswer((Answer<Void>) mock -> {
|
|||
|
|
++ AckCallback ackCallback = mock.getArgument(2);
|
|||
|
|
++ ackCallback.onSuccess(ackResult);
|
|||
|
|
++ return null;
|
|||
|
|
++ }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any());
|
|||
|
|
++
|
|||
|
|
++ assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testChangeInvisibleTimeAsync() throws Exception {
|
|||
|
|
+ AckResult ackResult = new AckResult();
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch013-backport-enhance-admin-output.patch b/patch013-backport-enhance-admin-output.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..3fa60916f
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch013-backport-enhance-admin-output.patch
|
|||
|
|
@@ -0,0 +1,892 @@
|
|||
|
|
+From 7e018520ef707a841c66c55d621f6560d03b631b Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Zhouxiang Zhan <zhouxzhan@apache.org>
|
|||
|
|
+Date: Fri, 25 Aug 2023 09:49:22 +0800
|
|||
|
|
+Subject: [PATCH 1/6] Add expireAfterAccess for cache (#7247)
|
|||
|
|
+
|
|||
|
|
+Add expireAfterAccess for cache
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/proxy/config/ProxyConfig.java | 59 ++++++++++++++-----
|
|||
|
|
+ .../metadata/ClusterMetadataService.java | 6 +-
|
|||
|
|
+ .../service/route/TopicRouteService.java | 14 +++--
|
|||
|
|
+ 3 files changed, 56 insertions(+), 23 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+index 76a243919..2994893d7 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+@@ -155,14 +155,17 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ private int consumerProcessorThreadPoolQueueCapacity = 10000;
|
|||
|
|
+
|
|||
|
|
+ private boolean useEndpointPortFromRequest = false;
|
|||
|
|
+- private int topicRouteServiceCacheExpiredInSeconds = 20;
|
|||
|
|
++
|
|||
|
|
++ private int topicRouteServiceCacheExpiredSeconds = 300;
|
|||
|
|
++ private int topicRouteServiceCacheRefreshSeconds = 20;
|
|||
|
|
+ private int topicRouteServiceCacheMaxNum = 20000;
|
|||
|
|
+ private int topicRouteServiceThreadPoolNums = PROCESSOR_NUMBER;
|
|||
|
|
+ private int topicRouteServiceThreadPoolQueueCapacity = 5000;
|
|||
|
|
+-
|
|||
|
|
+- private int topicConfigCacheExpiredInSeconds = 20;
|
|||
|
|
++ private int topicConfigCacheExpiredSeconds = 300;
|
|||
|
|
++ private int topicConfigCacheRefreshSeconds = 20;
|
|||
|
|
+ private int topicConfigCacheMaxNum = 20000;
|
|||
|
|
+- private int subscriptionGroupConfigCacheExpiredInSeconds = 20;
|
|||
|
|
++ private int subscriptionGroupConfigCacheExpiredSeconds = 300;
|
|||
|
|
++ private int subscriptionGroupConfigCacheRefreshSeconds = 20;
|
|||
|
|
+ private int subscriptionGroupConfigCacheMaxNum = 20000;
|
|||
|
|
+ private int metadataThreadPoolNums = 3;
|
|||
|
|
+ private int metadataThreadPoolQueueCapacity = 100000;
|
|||
|
|
+@@ -794,12 +797,20 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ this.consumerProcessorThreadPoolQueueCapacity = consumerProcessorThreadPoolQueueCapacity;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public int getTopicRouteServiceCacheExpiredInSeconds() {
|
|||
|
|
+- return topicRouteServiceCacheExpiredInSeconds;
|
|||
|
|
++ public int getTopicRouteServiceCacheExpiredSeconds() {
|
|||
|
|
++ return topicRouteServiceCacheExpiredSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setTopicRouteServiceCacheExpiredSeconds(int topicRouteServiceCacheExpiredSeconds) {
|
|||
|
|
++ this.topicRouteServiceCacheExpiredSeconds = topicRouteServiceCacheExpiredSeconds;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void setTopicRouteServiceCacheExpiredInSeconds(int topicRouteServiceCacheExpiredInSeconds) {
|
|||
|
|
+- this.topicRouteServiceCacheExpiredInSeconds = topicRouteServiceCacheExpiredInSeconds;
|
|||
|
|
++ public int getTopicRouteServiceCacheRefreshSeconds() {
|
|||
|
|
++ return topicRouteServiceCacheRefreshSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setTopicRouteServiceCacheRefreshSeconds(int topicRouteServiceCacheRefreshSeconds) {
|
|||
|
|
++ this.topicRouteServiceCacheRefreshSeconds = topicRouteServiceCacheRefreshSeconds;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public int getTopicRouteServiceCacheMaxNum() {
|
|||
|
|
+@@ -826,12 +837,20 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ this.topicRouteServiceThreadPoolQueueCapacity = topicRouteServiceThreadPoolQueueCapacity;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public int getTopicConfigCacheExpiredInSeconds() {
|
|||
|
|
+- return topicConfigCacheExpiredInSeconds;
|
|||
|
|
++ public int getTopicConfigCacheRefreshSeconds() {
|
|||
|
|
++ return topicConfigCacheRefreshSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setTopicConfigCacheRefreshSeconds(int topicConfigCacheRefreshSeconds) {
|
|||
|
|
++ this.topicConfigCacheRefreshSeconds = topicConfigCacheRefreshSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public int getTopicConfigCacheExpiredSeconds() {
|
|||
|
|
++ return topicConfigCacheExpiredSeconds;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void setTopicConfigCacheExpiredInSeconds(int topicConfigCacheExpiredInSeconds) {
|
|||
|
|
+- this.topicConfigCacheExpiredInSeconds = topicConfigCacheExpiredInSeconds;
|
|||
|
|
++ public void setTopicConfigCacheExpiredSeconds(int topicConfigCacheExpiredSeconds) {
|
|||
|
|
++ this.topicConfigCacheExpiredSeconds = topicConfigCacheExpiredSeconds;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public int getTopicConfigCacheMaxNum() {
|
|||
|
|
+@@ -842,12 +861,20 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ this.topicConfigCacheMaxNum = topicConfigCacheMaxNum;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public int getSubscriptionGroupConfigCacheExpiredInSeconds() {
|
|||
|
|
+- return subscriptionGroupConfigCacheExpiredInSeconds;
|
|||
|
|
++ public int getSubscriptionGroupConfigCacheRefreshSeconds() {
|
|||
|
|
++ return subscriptionGroupConfigCacheRefreshSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setSubscriptionGroupConfigCacheRefreshSeconds(int subscriptionGroupConfigCacheRefreshSeconds) {
|
|||
|
|
++ this.subscriptionGroupConfigCacheRefreshSeconds = subscriptionGroupConfigCacheRefreshSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public int getSubscriptionGroupConfigCacheExpiredSeconds() {
|
|||
|
|
++ return subscriptionGroupConfigCacheExpiredSeconds;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void setSubscriptionGroupConfigCacheExpiredInSeconds(int subscriptionGroupConfigCacheExpiredInSeconds) {
|
|||
|
|
+- this.subscriptionGroupConfigCacheExpiredInSeconds = subscriptionGroupConfigCacheExpiredInSeconds;
|
|||
|
|
++ public void setSubscriptionGroupConfigCacheExpiredSeconds(int subscriptionGroupConfigCacheExpiredSeconds) {
|
|||
|
|
++ this.subscriptionGroupConfigCacheExpiredSeconds = subscriptionGroupConfigCacheExpiredSeconds;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public int getSubscriptionGroupConfigCacheMaxNum() {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java
|
|||
|
|
+index bc9582ad8..d34a0efd9 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java
|
|||
|
|
+@@ -69,11 +69,13 @@ public class ClusterMetadataService extends AbstractStartAndShutdown implements
|
|||
|
|
+ );
|
|||
|
|
+ this.topicConfigCache = CacheBuilder.newBuilder()
|
|||
|
|
+ .maximumSize(config.getTopicConfigCacheMaxNum())
|
|||
|
|
+- .refreshAfterWrite(config.getTopicConfigCacheExpiredInSeconds(), TimeUnit.SECONDS)
|
|||
|
|
++ .expireAfterAccess(config.getTopicConfigCacheExpiredSeconds(), TimeUnit.SECONDS)
|
|||
|
|
++ .refreshAfterWrite(config.getTopicConfigCacheRefreshSeconds(), TimeUnit.SECONDS)
|
|||
|
|
+ .build(new ClusterTopicConfigCacheLoader());
|
|||
|
|
+ this.subscriptionGroupConfigCache = CacheBuilder.newBuilder()
|
|||
|
|
+ .maximumSize(config.getSubscriptionGroupConfigCacheMaxNum())
|
|||
|
|
+- .refreshAfterWrite(config.getSubscriptionGroupConfigCacheExpiredInSeconds(), TimeUnit.SECONDS)
|
|||
|
|
++ .expireAfterAccess(config.getSubscriptionGroupConfigCacheExpiredSeconds(), TimeUnit.SECONDS)
|
|||
|
|
++ .refreshAfterWrite(config.getSubscriptionGroupConfigCacheRefreshSeconds(), TimeUnit.SECONDS)
|
|||
|
|
+ .build(new ClusterSubscriptionGroupConfigCacheLoader());
|
|||
|
|
+
|
|||
|
|
+ this.init();
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
+index e012a5465..84348adc3 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
+@@ -68,10 +68,13 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown {
|
|||
|
|
+ );
|
|||
|
|
+ this.mqClientAPIFactory = mqClientAPIFactory;
|
|||
|
|
+
|
|||
|
|
+- this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum()).
|
|||
|
|
+- refreshAfterWrite(config.getTopicRouteServiceCacheExpiredInSeconds(), TimeUnit.SECONDS).
|
|||
|
|
+- executor(cacheRefreshExecutor).build(new CacheLoader<String, MessageQueueView>() {
|
|||
|
|
+- @Override public @Nullable MessageQueueView load(String topic) throws Exception {
|
|||
|
|
++ this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum())
|
|||
|
|
++ .expireAfterAccess(config.getTopicRouteServiceCacheExpiredSeconds(), TimeUnit.SECONDS)
|
|||
|
|
++ .refreshAfterWrite(config.getTopicRouteServiceCacheRefreshSeconds(), TimeUnit.SECONDS)
|
|||
|
|
++ .executor(cacheRefreshExecutor)
|
|||
|
|
++ .build(new CacheLoader<String, MessageQueueView>() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public @Nullable MessageQueueView load(String topic) throws Exception {
|
|||
|
|
+ try {
|
|||
|
|
+ TopicRouteData topicRouteData = mqClientAPIFactory.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis());
|
|||
|
|
+ return buildMessageQueueView(topic, topicRouteData);
|
|||
|
|
+@@ -83,7 +86,8 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- @Override public @Nullable MessageQueueView reload(@NonNull String key,
|
|||
|
|
++ @Override
|
|||
|
|
++ public @Nullable MessageQueueView reload(@NonNull String key,
|
|||
|
|
+ @NonNull MessageQueueView oldValue) throws Exception {
|
|||
|
|
+ try {
|
|||
|
|
+ return load(key);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 5f6dc90f9dab35809fcb0407d4d5cc2737d2335e Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ziyi Tan <tanziyi0925@gmail.com>
|
|||
|
|
+Date: Fri, 25 Aug 2023 11:17:23 +0800
|
|||
|
|
+Subject: [PATCH 2/6] [ISSUE #7250] Beautify command rocksDBConfigToJson output
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: Ziy1-Tan <ajb4596984460@gmail.com>
|
|||
|
|
+---
|
|||
|
|
+ .../metadata/RocksDBConfigToJsonCommand.java | 32 +++++++++++--------
|
|||
|
|
+ 1 file changed, 18 insertions(+), 14 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+index 3053f4684..3fc63e4dd 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+@@ -21,13 +21,13 @@ import org.apache.commons.cli.CommandLine;
|
|||
|
|
+ import org.apache.commons.cli.Option;
|
|||
|
|
+ import org.apache.commons.cli.Options;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
++import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
+ import org.apache.rocketmq.common.config.RocksDBConfigManager;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.DataConverter;
|
|||
|
|
+ import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.SubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.SubCommandException;
|
|||
|
|
+
|
|||
|
|
+-import java.io.File;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+
|
|||
|
|
+@@ -48,7 +48,7 @@ public class RocksDBConfigToJsonCommand implements SubCommand {
|
|||
|
|
+ @Override
|
|||
|
|
+ public Options buildCommandlineOptions(Options options) {
|
|||
|
|
+ Option pathOption = new Option("p", "path", true,
|
|||
|
|
+- "Absolute path to the metadata directory");
|
|||
|
|
++ "Absolute path for the metadata directory");
|
|||
|
|
+ pathOption.setRequired(true);
|
|||
|
|
+ options.addOption(pathOption);
|
|||
|
|
+
|
|||
|
|
+@@ -63,15 +63,14 @@ public class RocksDBConfigToJsonCommand implements SubCommand {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException {
|
|||
|
|
+ String path = commandLine.getOptionValue("path").trim();
|
|||
|
|
+- if (StringUtils.isEmpty(path) || !new File(path).exists()) {
|
|||
|
|
++ if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) {
|
|||
|
|
+ System.out.print("Rocksdb path is invalid.\n");
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ String configType = commandLine.getOptionValue("configType").trim().toLowerCase();
|
|||
|
|
+
|
|||
|
|
+- final long memTableFlushInterval = 60 * 60 * 1000L;
|
|||
|
|
+- RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval);
|
|||
|
|
++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(60 * 60 * 1000L);
|
|||
|
|
+ try {
|
|||
|
|
+ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
+ // for topics.json
|
|||
|
|
+@@ -84,13 +83,16 @@ public class RocksDBConfigToJsonCommand implements SubCommand {
|
|||
|
|
+ topicConfigTable.put(topic, jsonObject);
|
|||
|
|
+ });
|
|||
|
|
+
|
|||
|
|
+- if (isLoad) {
|
|||
|
|
+- topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable));
|
|||
|
|
+- final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true);
|
|||
|
|
+- System.out.print(topicsJsonStr + "\n");
|
|||
|
|
++ if (!isLoad) {
|
|||
|
|
++ System.out.print("RocksDB load error, path=" + path);
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable));
|
|||
|
|
++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true);
|
|||
|
|
++ System.out.print(topicsJsonStr + "\n");
|
|||
|
|
++ return;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
+ // for subscriptionGroup.json
|
|||
|
|
+ final Map<String, JSONObject> subscriptionGroupJsonConfig = new HashMap<>();
|
|||
|
|
+@@ -102,13 +104,15 @@ public class RocksDBConfigToJsonCommand implements SubCommand {
|
|||
|
|
+ subscriptionGroupTable.put(subscriptionGroup, jsonObject);
|
|||
|
|
+ });
|
|||
|
|
+
|
|||
|
|
+- if (isLoad) {
|
|||
|
|
+- subscriptionGroupJsonConfig.put("subscriptionGroupTable",
|
|||
|
|
+- (JSONObject) JSONObject.toJSON(subscriptionGroupTable));
|
|||
|
|
+- final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true);
|
|||
|
|
+- System.out.print(subscriptionGroupJsonStr + "\n");
|
|||
|
|
++ if (!isLoad) {
|
|||
|
|
++ System.out.print("RocksDB load error, path=" + path);
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
++ subscriptionGroupJsonConfig.put("subscriptionGroupTable",
|
|||
|
|
++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable));
|
|||
|
|
++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true);
|
|||
|
|
++ System.out.print(subscriptionGroupJsonStr + "\n");
|
|||
|
|
++ return;
|
|||
|
|
+ }
|
|||
|
|
+ System.out.print("Config type was not recognized, configType=" + configType + "\n");
|
|||
|
|
+ } finally {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From b4f73e2aabc1b141cec98431899e4090340adf0f Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: mxsm <ljbmxsm@gmail.com>
|
|||
|
|
+Date: Sun, 27 Aug 2023 20:58:58 +0800
|
|||
|
|
+Subject: [PATCH 3/6] [ISSUE #7271] Optimize the configuration for setting the
|
|||
|
|
+ quantity of TimerDequeuePutMessageService (#7272)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../java/org/apache/rocketmq/store/timer/TimerMessageStore.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
+index 690f4863e..181f7087a 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
+@@ -222,7 +222,7 @@ public class TimerMessageStore {
|
|||
|
|
+ dequeueGetMessageServices[i] = new TimerDequeueGetMessageService();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- int putThreadNum = Math.max(storeConfig.getTimerGetMessageThreadNum(), 1);
|
|||
|
|
++ int putThreadNum = Math.max(storeConfig.getTimerPutMessageThreadNum(), 1);
|
|||
|
|
+ dequeuePutMessageServices = new TimerDequeuePutMessageService[putThreadNum];
|
|||
|
|
+ for (int i = 0; i < dequeuePutMessageServices.length; i++) {
|
|||
|
|
+ dequeuePutMessageServices[i] = new TimerDequeuePutMessageService();
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 3e100103af68588528bf32f3752a85e8023f46f8 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ziyi Tan <tanziyi0925@gmail.com>
|
|||
|
|
+Date: Tue, 29 Aug 2023 13:48:51 +0800
|
|||
|
|
+Subject: [PATCH 4/6] [ISSUE #7277] Enhance rocksDBConfigToJson to support
|
|||
|
|
+ metadata counting (#7276)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../common/config/AbstractRocksDBStorage.java | 4 +-
|
|||
|
|
+ .../common/config/ConfigRocksDBStorage.java | 6 +
|
|||
|
|
+ .../tools/command/MQAdminStartup.java | 4 +-
|
|||
|
|
+ .../ExportMetadataInRocksDBCommand.java | 138 ++++++++++++++++++
|
|||
|
|
+ .../metadata/RocksDBConfigToJsonCommand.java | 122 ----------------
|
|||
|
|
+ ...> ExportMetadataInRocksDBCommandTest.java} | 38 +++--
|
|||
|
|
+ 6 files changed, 173 insertions(+), 139 deletions(-)
|
|||
|
|
+ create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java
|
|||
|
|
+ delete mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+ rename tools/src/test/java/org/apache/rocketmq/tools/command/metadata/{KvConfigToJsonCommandTest.java => ExportMetadataInRocksDBCommandTest.java} (62%)
|
|||
|
|
+
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
|
|||
|
|
+index e3673baad..a720a5be3 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
|
|||
|
|
+@@ -385,8 +385,10 @@ public abstract class AbstractRocksDBStorage {
|
|||
|
|
+ this.options.close();
|
|||
|
|
+ }
|
|||
|
|
+ //4. close db.
|
|||
|
|
+- if (db != null) {
|
|||
|
|
++ if (db != null && !this.readOnly) {
|
|||
|
|
+ this.db.syncWal();
|
|||
|
|
++ }
|
|||
|
|
++ if (db != null) {
|
|||
|
|
+ this.db.closeE();
|
|||
|
|
+ }
|
|||
|
|
+ //5. help gc.
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java
|
|||
|
|
+index 9d05ed282..463bd8fed 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java
|
|||
|
|
+@@ -60,6 +60,12 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage {
|
|||
|
|
+ this.readOnly = false;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public ConfigRocksDBStorage(final String dbPath, boolean readOnly) {
|
|||
|
|
++ super();
|
|||
|
|
++ this.dbPath = dbPath;
|
|||
|
|
++ this.readOnly = readOnly;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ private void initOptions() {
|
|||
|
|
+ this.options = createConfigDBOptions();
|
|||
|
|
+
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
+index 324aa1856..788fa83c2 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
|
|||
|
|
+@@ -80,7 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.message.SendMessageCommand;
|
|||
|
|
+-import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand;
|
|||
|
|
++import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand;
|
|||
|
|
+@@ -212,7 +212,6 @@ public class MQAdminStartup {
|
|||
|
|
+
|
|||
|
|
+ initCommand(new ClusterListSubCommand());
|
|||
|
|
+ initCommand(new TopicListSubCommand());
|
|||
|
|
+- initCommand(new RocksDBConfigToJsonCommand());
|
|||
|
|
+
|
|||
|
|
+ initCommand(new UpdateKvConfigCommand());
|
|||
|
|
+ initCommand(new DeleteKvConfigCommand());
|
|||
|
|
+@@ -257,6 +256,7 @@ public class MQAdminStartup {
|
|||
|
|
+ initCommand(new ExportMetadataCommand());
|
|||
|
|
+ initCommand(new ExportConfigsCommand());
|
|||
|
|
+ initCommand(new ExportMetricsCommand());
|
|||
|
|
++ initCommand(new ExportMetadataInRocksDBCommand());
|
|||
|
|
+
|
|||
|
|
+ initCommand(new HAStatusSubCommand());
|
|||
|
|
+
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..2a7d3fba4
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java
|
|||
|
|
+@@ -0,0 +1,138 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++package org.apache.rocketmq.tools.command.export;
|
|||
|
|
++
|
|||
|
|
++import com.alibaba.fastjson.JSONObject;
|
|||
|
|
++import org.apache.commons.cli.CommandLine;
|
|||
|
|
++import org.apache.commons.cli.Option;
|
|||
|
|
++import org.apache.commons.cli.Options;
|
|||
|
|
++import org.apache.commons.lang3.StringUtils;
|
|||
|
|
++import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
++import org.apache.rocketmq.common.config.ConfigRocksDBStorage;
|
|||
|
|
++import org.apache.rocketmq.common.utils.DataConverter;
|
|||
|
|
++import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
++import org.apache.rocketmq.tools.command.SubCommand;
|
|||
|
|
++import org.apache.rocketmq.tools.command.SubCommandException;
|
|||
|
|
++import org.rocksdb.RocksIterator;
|
|||
|
|
++
|
|||
|
|
++import java.util.HashMap;
|
|||
|
|
++import java.util.Map;
|
|||
|
|
++import java.util.concurrent.atomic.AtomicLong;
|
|||
|
|
++import java.util.function.BiConsumer;
|
|||
|
|
++
|
|||
|
|
++public class ExportMetadataInRocksDBCommand implements SubCommand {
|
|||
|
|
++ private static final String TOPICS_JSON_CONFIG = "topics";
|
|||
|
|
++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups";
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public String commandName() {
|
|||
|
|
++ return "exportMetadataInRocksDB";
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public String commandDesc() {
|
|||
|
|
++ return "export RocksDB kv config (topics/subscriptionGroups)";
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public Options buildCommandlineOptions(Options options) {
|
|||
|
|
++ Option pathOption = new Option("p", "path", true,
|
|||
|
|
++ "Absolute path for the metadata directory");
|
|||
|
|
++ pathOption.setRequired(true);
|
|||
|
|
++ options.addOption(pathOption);
|
|||
|
|
++
|
|||
|
|
++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " +
|
|||
|
|
++ "topics/subscriptionGroups");
|
|||
|
|
++ configTypeOption.setRequired(true);
|
|||
|
|
++ options.addOption(configTypeOption);
|
|||
|
|
++
|
|||
|
|
++ Option jsonEnableOption = new Option("j", "jsonEnable", true,
|
|||
|
|
++ "Json format enable, Default: false");
|
|||
|
|
++ jsonEnableOption.setRequired(false);
|
|||
|
|
++ options.addOption(jsonEnableOption);
|
|||
|
|
++
|
|||
|
|
++ return options;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException {
|
|||
|
|
++ String path = commandLine.getOptionValue("path").trim();
|
|||
|
|
++ if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) {
|
|||
|
|
++ System.out.print("RocksDB path is invalid.\n");
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase();
|
|||
|
|
++
|
|||
|
|
++ boolean jsonEnable = false;
|
|||
|
|
++ if (commandLine.hasOption("jsonEnable")) {
|
|||
|
|
++ jsonEnable = Boolean.parseBoolean(commandLine.getOptionValue("jsonEnable").trim());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++
|
|||
|
|
++ ConfigRocksDBStorage kvStore = new ConfigRocksDBStorage(path, true /* readOnly */);
|
|||
|
|
++ if (!kvStore.start()) {
|
|||
|
|
++ System.out.print("RocksDB load error, path=" + path + "\n");
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ try {
|
|||
|
|
++ if (TOPICS_JSON_CONFIG.equalsIgnoreCase(configType) || SUBSCRIPTION_GROUP_JSON_CONFIG.equalsIgnoreCase(configType)) {
|
|||
|
|
++ handleExportMetadata(kvStore, configType, jsonEnable);
|
|||
|
|
++ } else {
|
|||
|
|
++ System.out.printf("Invalid config type=%s, Options: topics,subscriptionGroups\n", configType);
|
|||
|
|
++ }
|
|||
|
|
++ } finally {
|
|||
|
|
++ kvStore.shutdown();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private static void handleExportMetadata(ConfigRocksDBStorage kvStore, String configType, boolean jsonEnable) {
|
|||
|
|
++ if (jsonEnable) {
|
|||
|
|
++ final Map<String, JSONObject> jsonConfig = new HashMap<>();
|
|||
|
|
++ final Map<String, JSONObject> configTable = new HashMap<>();
|
|||
|
|
++ iterateKvStore(kvStore, (key, value) -> {
|
|||
|
|
++ final String configKey = new String(key, DataConverter.charset);
|
|||
|
|
++ final String configValue = new String(value, DataConverter.charset);
|
|||
|
|
++ final JSONObject jsonObject = JSONObject.parseObject(configValue);
|
|||
|
|
++ configTable.put(configKey, jsonObject);
|
|||
|
|
++ }
|
|||
|
|
++ );
|
|||
|
|
++
|
|||
|
|
++ jsonConfig.put(configType.equalsIgnoreCase(TOPICS_JSON_CONFIG) ? "topicConfigTable" : "subscriptionGroupTable",
|
|||
|
|
++ (JSONObject) JSONObject.toJSON(configTable));
|
|||
|
|
++ final String jsonConfigStr = JSONObject.toJSONString(jsonConfig, true);
|
|||
|
|
++ System.out.print(jsonConfigStr + "\n");
|
|||
|
|
++ } else {
|
|||
|
|
++ AtomicLong count = new AtomicLong(0);
|
|||
|
|
++ iterateKvStore(kvStore, (key, value) -> {
|
|||
|
|
++ final String configKey = new String(key, DataConverter.charset);
|
|||
|
|
++ final String configValue = new String(value, DataConverter.charset);
|
|||
|
|
++ System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), configKey, configValue);
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private static void iterateKvStore(ConfigRocksDBStorage kvStore, BiConsumer<byte[], byte[]> biConsumer) {
|
|||
|
|
++ try (RocksIterator iterator = kvStore.iterator()) {
|
|||
|
|
++ iterator.seekToFirst();
|
|||
|
|
++ for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {
|
|||
|
|
++ biConsumer.accept(iterator.key(), iterator.value());
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
+deleted file mode 100644
|
|||
|
|
+index 3fc63e4dd..000000000
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
++++ /dev/null
|
|||
|
|
+@@ -1,122 +0,0 @@
|
|||
|
|
+-/*
|
|||
|
|
+- * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+- * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+- * this work for additional information regarding copyright ownership.
|
|||
|
|
+- * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+- * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+- * the License. You may obtain a copy of the License at
|
|||
|
|
+- *
|
|||
|
|
+- * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+- *
|
|||
|
|
+- * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+- * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+- * See the License for the specific language governing permissions and
|
|||
|
|
+- * limitations under the License.
|
|||
|
|
+- */
|
|||
|
|
+-package org.apache.rocketmq.tools.command.metadata;
|
|||
|
|
+-
|
|||
|
|
+-import com.alibaba.fastjson.JSONObject;
|
|||
|
|
+-import org.apache.commons.cli.CommandLine;
|
|||
|
|
+-import org.apache.commons.cli.Option;
|
|||
|
|
+-import org.apache.commons.cli.Options;
|
|||
|
|
+-import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+-import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
+-import org.apache.rocketmq.common.config.RocksDBConfigManager;
|
|||
|
|
+-import org.apache.rocketmq.common.utils.DataConverter;
|
|||
|
|
+-import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+-import org.apache.rocketmq.tools.command.SubCommand;
|
|||
|
|
+-import org.apache.rocketmq.tools.command.SubCommandException;
|
|||
|
|
+-
|
|||
|
|
+-import java.util.HashMap;
|
|||
|
|
+-import java.util.Map;
|
|||
|
|
+-
|
|||
|
|
+-public class RocksDBConfigToJsonCommand implements SubCommand {
|
|||
|
|
+- private static final String TOPICS_JSON_CONFIG = "topics";
|
|||
|
|
+- private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups";
|
|||
|
|
+-
|
|||
|
|
+- @Override
|
|||
|
|
+- public String commandName() {
|
|||
|
|
+- return "rocksDBConfigToJson";
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- @Override
|
|||
|
|
+- public String commandDesc() {
|
|||
|
|
+- return "Convert RocksDB kv config (topics/subscriptionGroups) to json";
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- @Override
|
|||
|
|
+- public Options buildCommandlineOptions(Options options) {
|
|||
|
|
+- Option pathOption = new Option("p", "path", true,
|
|||
|
|
+- "Absolute path for the metadata directory");
|
|||
|
|
+- pathOption.setRequired(true);
|
|||
|
|
+- options.addOption(pathOption);
|
|||
|
|
+-
|
|||
|
|
+- Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " +
|
|||
|
|
+- "topics/subscriptionGroups");
|
|||
|
|
+- configTypeOption.setRequired(true);
|
|||
|
|
+- options.addOption(configTypeOption);
|
|||
|
|
+-
|
|||
|
|
+- return options;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- @Override
|
|||
|
|
+- public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException {
|
|||
|
|
+- String path = commandLine.getOptionValue("path").trim();
|
|||
|
|
+- if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) {
|
|||
|
|
+- System.out.print("Rocksdb path is invalid.\n");
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- String configType = commandLine.getOptionValue("configType").trim().toLowerCase();
|
|||
|
|
+-
|
|||
|
|
+- RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(60 * 60 * 1000L);
|
|||
|
|
+- try {
|
|||
|
|
+- if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
+- // for topics.json
|
|||
|
|
+- final Map<String, JSONObject> topicsJsonConfig = new HashMap<>();
|
|||
|
|
+- final Map<String, JSONObject> topicConfigTable = new HashMap<>();
|
|||
|
|
+- boolean isLoad = kvConfigManager.load(path, (key, value) -> {
|
|||
|
|
+- final String topic = new String(key, DataConverter.charset);
|
|||
|
|
+- final String topicConfig = new String(value, DataConverter.charset);
|
|||
|
|
+- final JSONObject jsonObject = JSONObject.parseObject(topicConfig);
|
|||
|
|
+- topicConfigTable.put(topic, jsonObject);
|
|||
|
|
+- });
|
|||
|
|
+-
|
|||
|
|
+- if (!isLoad) {
|
|||
|
|
+- System.out.print("RocksDB load error, path=" + path);
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable));
|
|||
|
|
+- final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true);
|
|||
|
|
+- System.out.print(topicsJsonStr + "\n");
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
+- // for subscriptionGroup.json
|
|||
|
|
+- final Map<String, JSONObject> subscriptionGroupJsonConfig = new HashMap<>();
|
|||
|
|
+- final Map<String, JSONObject> subscriptionGroupTable = new HashMap<>();
|
|||
|
|
+- boolean isLoad = kvConfigManager.load(path, (key, value) -> {
|
|||
|
|
+- final String subscriptionGroup = new String(key, DataConverter.charset);
|
|||
|
|
+- final String subscriptionGroupConfig = new String(value, DataConverter.charset);
|
|||
|
|
+- final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig);
|
|||
|
|
+- subscriptionGroupTable.put(subscriptionGroup, jsonObject);
|
|||
|
|
+- });
|
|||
|
|
+-
|
|||
|
|
+- if (!isLoad) {
|
|||
|
|
+- System.out.print("RocksDB load error, path=" + path);
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- subscriptionGroupJsonConfig.put("subscriptionGroupTable",
|
|||
|
|
+- (JSONObject) JSONObject.toJSON(subscriptionGroupTable));
|
|||
|
|
+- final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true);
|
|||
|
|
+- System.out.print(subscriptionGroupJsonStr + "\n");
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- System.out.print("Config type was not recognized, configType=" + configType + "\n");
|
|||
|
|
+- } finally {
|
|||
|
|
+- kvConfigManager.stop();
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+-}
|
|||
|
|
+diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java
|
|||
|
|
+similarity index 62%
|
|||
|
|
+rename from tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java
|
|||
|
|
+rename to tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java
|
|||
|
|
+index b2f66c7b0..2b938c90f 100644
|
|||
|
|
+--- a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java
|
|||
|
|
++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java
|
|||
|
|
+@@ -21,43 +21,53 @@ import org.apache.commons.cli.DefaultParser;
|
|||
|
|
+ import org.apache.commons.cli.Options;
|
|||
|
|
+ import org.apache.rocketmq.srvutil.ServerUtil;
|
|||
|
|
+ import org.apache.rocketmq.tools.command.SubCommandException;
|
|||
|
|
++import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
+ import java.io.File;
|
|||
|
|
+
|
|||
|
|
+ import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
+
|
|||
|
|
+-public class KvConfigToJsonCommandTest {
|
|||
|
|
++public class ExportMetadataInRocksDBCommandTest {
|
|||
|
|
+ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/";
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testExecute() throws SubCommandException {
|
|||
|
|
+ {
|
|||
|
|
+- String[] cases = new String[]{"topics", "subscriptionGroups"};
|
|||
|
|
+- for (String c : cases) {
|
|||
|
|
+- RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand();
|
|||
|
|
++ String[][] cases = new String[][] {
|
|||
|
|
++ {"topics", "false"},
|
|||
|
|
++ {"topics", "false1"},
|
|||
|
|
++ {"topics", "true"},
|
|||
|
|
++ {"subscriptionGroups", "false"},
|
|||
|
|
++ {"subscriptionGroups", "false2"},
|
|||
|
|
++ {"subscriptionGroups", "true"}
|
|||
|
|
++ };
|
|||
|
|
++
|
|||
|
|
++ for (String[] c : cases) {
|
|||
|
|
++ ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand();
|
|||
|
|
+ Options options = ServerUtil.buildCommandlineOptions(new Options());
|
|||
|
|
+- String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c};
|
|||
|
|
++ String[] subargs = new String[] {"-p " + BASE_PATH + c[0], "-t " + c[0], "-j " + c[1]};
|
|||
|
|
+ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs,
|
|||
|
|
+- cmd.buildCommandlineOptions(options), new DefaultParser());
|
|||
|
|
++ cmd.buildCommandlineOptions(options), new DefaultParser());
|
|||
|
|
+ cmd.execute(commandLine, options, null);
|
|||
|
|
+- assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c);
|
|||
|
|
+- assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c);
|
|||
|
|
++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c[0]);
|
|||
|
|
++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c[0]);
|
|||
|
|
++ assertThat(commandLine.getOptionValue("j").trim()).isEqualTo(c[1]);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ // invalid cases
|
|||
|
|
+ {
|
|||
|
|
+- String[][] cases = new String[][]{
|
|||
|
|
+- {"-p " + BASE_PATH + "tmpPath", "-t topics"},
|
|||
|
|
+- {"-p ", "-t topics"},
|
|||
|
|
+- {"-p " + BASE_PATH + "topics", "-t invalid_type"}
|
|||
|
|
++ String[][] cases = new String[][] {
|
|||
|
|
++ {"-p " + BASE_PATH + "tmpPath", "-t topics", "-j true"},
|
|||
|
|
++ {"-p ", "-t topics", "-j true"},
|
|||
|
|
++ {"-p " + BASE_PATH + "topics", "-t invalid_type", "-j true"}
|
|||
|
|
+ };
|
|||
|
|
+
|
|||
|
|
+ for (String[] c : cases) {
|
|||
|
|
+- RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand();
|
|||
|
|
++ ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand();
|
|||
|
|
+ Options options = ServerUtil.buildCommandlineOptions(new Options());
|
|||
|
|
+ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c,
|
|||
|
|
+- cmd.buildCommandlineOptions(options), new DefaultParser());
|
|||
|
|
++ cmd.buildCommandlineOptions(options), new DefaultParser());
|
|||
|
|
+ cmd.execute(commandLine, options, null);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From fa549154370cb866a90e37c13a90d2c598d6b1f6 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yuz10 <845238369@qq.com>
|
|||
|
|
+Date: Tue, 29 Aug 2023 15:22:09 +0800
|
|||
|
|
+Subject: [PATCH 5/6] [ISSUE #7261] Slave high CPU usage when
|
|||
|
|
+ enableScheduleAsyncDeliver=true (#7262)
|
|||
|
|
+
|
|||
|
|
+* [ISSUE #6390] Add break to the exception of WHEEL_TIMER_NOT_ENABLE.
|
|||
|
|
+
|
|||
|
|
+* fix broker start fail if mapped file size is 0
|
|||
|
|
+
|
|||
|
|
+* log
|
|||
|
|
+
|
|||
|
|
+* only delete the last empty file
|
|||
|
|
+
|
|||
|
|
+* change dataReadAheadEnable default to true
|
|||
|
|
+
|
|||
|
|
+* fix endless loop when master change to slave.
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/broker/schedule/ScheduleMessageService.java | 7 ++++++-
|
|||
|
|
+ 1 file changed, 6 insertions(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
|
|||
|
|
+index aed0ee19f..297b14207 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
|
|||
|
|
+@@ -566,7 +566,8 @@ public class ScheduleMessageService extends ConfigManager {
|
|||
|
|
+ pendingQueue.remove();
|
|||
|
|
+ break;
|
|||
|
|
+ case RUNNING:
|
|||
|
|
+- break;
|
|||
|
|
++ scheduleNextTask();
|
|||
|
|
++ return;
|
|||
|
|
+ case EXCEPTION:
|
|||
|
|
+ if (!isStarted()) {
|
|||
|
|
+ log.warn("HandlePutResultTask shutdown, info={}", putResultProcess.toString());
|
|||
|
|
+@@ -586,6 +587,10 @@ public class ScheduleMessageService extends ConfigManager {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ scheduleNextTask();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private void scheduleNextTask() {
|
|||
|
|
+ if (isStarted()) {
|
|||
|
|
+ ScheduleMessageService.this.handleExecutorService
|
|||
|
|
+ .schedule(new HandlePutResultTask(this.delayLevel), DELAY_FOR_A_SLEEP, TimeUnit.MILLISECONDS);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 9f34f55e1dac495730c9cd5469f2ab3225b8f0b9 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: ShuangxiDing <dingshuangxi888@gmail.com>
|
|||
|
|
+Date: Tue, 29 Aug 2023 15:48:46 +0800
|
|||
|
|
+Subject: [PATCH 6/6] [ISSUE #7226] Filter tlvs in ppv2 which contents not are
|
|||
|
|
+ spec-compliant ASCII characters and space (#7227)
|
|||
|
|
+
|
|||
|
|
+Filter tlvs in ppv2 which not are spec-compliant ASCII characters and space
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/common/utils/BinaryUtil.java | 17 +++++++++++++++++
|
|||
|
|
+ .../grpc/ProxyAndTlsProtocolNegotiator.java | 8 +++++++-
|
|||
|
|
+ .../remoting/netty/NettyRemotingServer.java | 8 +++++++-
|
|||
|
|
+ 3 files changed, 31 insertions(+), 2 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java
|
|||
|
|
+index 421adaca4..7b4b24819 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java
|
|||
|
|
+@@ -43,4 +43,21 @@ public class BinaryUtil {
|
|||
|
|
+ byte[] bytes = calculateMd5(content);
|
|||
|
|
+ return Hex.encodeHexString(bytes, false);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * Returns true if subject contains only bytes that are spec-compliant ASCII characters.
|
|||
|
|
++ * @param subject
|
|||
|
|
++ * @return
|
|||
|
|
++ */
|
|||
|
|
++ public static boolean isAscii(byte[] subject) {
|
|||
|
|
++ if (subject == null) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++ for (byte b : subject) {
|
|||
|
|
++ if ((b & 0x80) != 0) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ return true;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+\ No newline at end of file
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java
|
|||
|
|
+index ee167bd7b..b584ddfbd 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java
|
|||
|
|
+@@ -24,6 +24,7 @@ import io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiator;
|
|||
|
|
+ import io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiators;
|
|||
|
|
+ import io.grpc.netty.shaded.io.grpc.netty.ProtocolNegotiationEvent;
|
|||
|
|
+ import io.grpc.netty.shaded.io.netty.buffer.ByteBuf;
|
|||
|
|
++import io.grpc.netty.shaded.io.netty.buffer.ByteBufUtil;
|
|||
|
|
+ import io.grpc.netty.shaded.io.netty.channel.ChannelHandler;
|
|||
|
|
+ import io.grpc.netty.shaded.io.netty.channel.ChannelHandlerContext;
|
|||
|
|
+ import io.grpc.netty.shaded.io.netty.channel.ChannelInboundHandlerAdapter;
|
|||
|
|
+@@ -44,6 +45,7 @@ import org.apache.commons.collections.CollectionUtils;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.HAProxyConstants;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.BinaryUtil;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+@@ -191,9 +193,13 @@ public class ProxyAndTlsProtocolNegotiator implements InternalProtocolNegotiator
|
|||
|
|
+ }
|
|||
|
|
+ if (CollectionUtils.isNotEmpty(msg.tlvs())) {
|
|||
|
|
+ msg.tlvs().forEach(tlv -> {
|
|||
|
|
++ byte[] valueBytes = ByteBufUtil.getBytes(tlv.content());
|
|||
|
|
++ if (!BinaryUtil.isAscii(valueBytes)) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
+ Attributes.Key<String> key = AttributeKeys.valueOf(
|
|||
|
|
+ HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue()));
|
|||
|
|
+- String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8));
|
|||
|
|
++ String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8));
|
|||
|
|
+ builder.set(key, value);
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
+index 17f138f86..e626260c9 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
+@@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty;
|
|||
|
|
+
|
|||
|
|
+ import io.netty.bootstrap.ServerBootstrap;
|
|||
|
|
+ import io.netty.buffer.ByteBuf;
|
|||
|
|
++import io.netty.buffer.ByteBufUtil;
|
|||
|
|
+ import io.netty.buffer.PooledByteBufAllocator;
|
|||
|
|
+ import io.netty.channel.Channel;
|
|||
|
|
+ import io.netty.channel.ChannelDuplexHandler;
|
|||
|
|
+@@ -58,6 +59,7 @@ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.HAProxyConstants;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.BinaryUtil;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.NetworkUtil;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+@@ -787,9 +789,13 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ }
|
|||
|
|
+ if (CollectionUtils.isNotEmpty(msg.tlvs())) {
|
|||
|
|
+ msg.tlvs().forEach(tlv -> {
|
|||
|
|
++ byte[] valueBytes = ByteBufUtil.getBytes(tlv.content());
|
|||
|
|
++ if (!BinaryUtil.isAscii(valueBytes)) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
+ AttributeKey<String> key = AttributeKeys.valueOf(
|
|||
|
|
+ HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue()));
|
|||
|
|
+- String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8));
|
|||
|
|
++ String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8));
|
|||
|
|
+ channel.attr(key).set(value);
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch014-backport-Queue-Selection-Strategy-Optimization.patch b/patch014-backport-Queue-Selection-Strategy-Optimization.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..90d3553aa
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch014-backport-Queue-Selection-Strategy-Optimization.patch
|
|||
|
|
@@ -0,0 +1,2023 @@
|
|||
|
|
+From b028277018946868838a82a08211071bc231a175 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ji Juntao <juntao.jjt@alibaba-inc.com>
|
|||
|
|
+Date: Tue, 29 Aug 2023 16:13:38 +0800
|
|||
|
|
+Subject: [PATCH] [ISSUE #6567] [RIP-63] Queue Selection Strategy Optimization
|
|||
|
|
+ (#6568)
|
|||
|
|
+
|
|||
|
|
+Optimize the proxy's and client's selection strategy for brokers when sending messages, and use multiple selection strategies as a pipeline to filter suitable queues.
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/client/ClientConfig.java | 54 +++++
|
|||
|
|
+ .../client/common/ThreadLocalIndex.java | 8 +
|
|||
|
|
+ .../rocketmq/client/impl/MQClientAPIImpl.java | 12 +-
|
|||
|
|
+ .../client/impl/factory/MQClientInstance.java | 7 +
|
|||
|
|
+ .../impl/producer/DefaultMQProducerImpl.java | 87 ++++++--
|
|||
|
|
+ .../impl/producer/TopicPublishInfo.java | 40 ++++
|
|||
|
|
+ .../client/latency/LatencyFaultTolerance.java | 66 +++++-
|
|||
|
|
+ .../latency/LatencyFaultToleranceImpl.java | 189 ++++++++++++++----
|
|||
|
|
+ .../client/latency/MQFaultStrategy.java | 155 ++++++++++----
|
|||
|
|
+ .../rocketmq/client/latency/Resolver.java | 17 +-
|
|||
|
|
+ .../client/latency/ServiceDetector.java | 30 +++
|
|||
|
|
+ .../LatencyFaultToleranceImplTest.java | 36 +++-
|
|||
|
|
+ .../processor/DefaultRequestProcessor.java | 24 ---
|
|||
|
|
+ .../rocketmq/proxy/config/ProxyConfig.java | 46 +++++
|
|||
|
|
+ .../grpc/v2/producer/SendMessageActivity.java | 2 +-
|
|||
|
|
+ .../proxy/processor/ProducerProcessor.java | 18 +-
|
|||
|
|
+ .../service/route/LocalTopicRouteService.java | 2 +-
|
|||
|
|
+ .../service/route/MessageQueueSelector.java | 95 ++++++++-
|
|||
|
|
+ .../proxy/service/route/MessageQueueView.java | 18 +-
|
|||
|
|
+ .../service/route/TopicRouteService.java | 80 +++++++-
|
|||
|
|
+ .../consumer/ReceiveMessageActivityTest.java | 5 +-
|
|||
|
|
+ .../v2/producer/SendMessageActivityTest.java | 82 +++++++-
|
|||
|
|
+ .../proxy/service/BaseServiceTest.java | 4 +-
|
|||
|
|
+ .../route/MessageQueueSelectorTest.java | 8 +-
|
|||
|
|
+ .../sysmessage/HeartbeatSyncerTest.java | 2 +-
|
|||
|
|
+ .../ClusterTransactionServiceTest.java | 8 +-
|
|||
|
|
+ 26 files changed, 919 insertions(+), 176 deletions(-)
|
|||
|
|
+ rename remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java => client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java (65%)
|
|||
|
|
+ create mode 100644 client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java
|
|||
|
|
+
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
|
|||
|
|
+index f87450f66..bb0fe3522 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
|
|||
|
|
+@@ -38,6 +38,8 @@ public class ClientConfig {
|
|||
|
|
+ public static final String SOCKS_PROXY_CONFIG = "com.rocketmq.socks.proxy.config";
|
|||
|
|
+ public static final String DECODE_READ_BODY = "com.rocketmq.read.body";
|
|||
|
|
+ public static final String DECODE_DECOMPRESS_BODY = "com.rocketmq.decompress.body";
|
|||
|
|
++ public static final String SEND_LATENCY_ENABLE = "com.rocketmq.sendLatencyEnable";
|
|||
|
|
++ public static final String START_DETECTOR_ENABLE = "com.rocketmq.startDetectorEnable";
|
|||
|
|
+ public static final String HEART_BEAT_V2 = "com.rocketmq.heartbeat.v2";
|
|||
|
|
+ private String namesrvAddr = NameServerAddressUtils.getNameServerAddresses();
|
|||
|
|
+ private String clientIP = NetworkUtil.getLocalAddress();
|
|||
|
|
+@@ -72,6 +74,8 @@ public class ClientConfig {
|
|||
|
|
+ private String socksProxyConfig = System.getProperty(SOCKS_PROXY_CONFIG, "{}");
|
|||
|
|
+
|
|||
|
|
+ private int mqClientApiTimeout = 3 * 1000;
|
|||
|
|
++ private int detectTimeout = 200;
|
|||
|
|
++ private int detectInterval = 2 * 1000;
|
|||
|
|
+
|
|||
|
|
+ private LanguageCode language = LanguageCode.JAVA;
|
|||
|
|
+
|
|||
|
|
+@@ -81,6 +85,15 @@ public class ClientConfig {
|
|||
|
|
+ */
|
|||
|
|
+ protected boolean enableStreamRequestType = false;
|
|||
|
|
+
|
|||
|
|
++ /**
|
|||
|
|
++ * Enable the fault tolerance mechanism of the client sending process.
|
|||
|
|
++ * DO NOT OPEN when ORDER messages are required.
|
|||
|
|
++ * Turning on will interfere with the queue selection functionality,
|
|||
|
|
++ * possibly conflicting with the order message.
|
|||
|
|
++ */
|
|||
|
|
++ private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false"));
|
|||
|
|
++ private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false"));
|
|||
|
|
++
|
|||
|
|
+ public String buildMQClientId() {
|
|||
|
|
+ StringBuilder sb = new StringBuilder();
|
|||
|
|
+ sb.append(this.getClientIP());
|
|||
|
|
+@@ -186,6 +199,10 @@ public class ClientConfig {
|
|||
|
|
+ this.decodeDecompressBody = cc.decodeDecompressBody;
|
|||
|
|
+ this.enableStreamRequestType = cc.enableStreamRequestType;
|
|||
|
|
+ this.useHeartbeatV2 = cc.useHeartbeatV2;
|
|||
|
|
++ this.startDetectorEnable = cc.startDetectorEnable;
|
|||
|
|
++ this.sendLatencyEnable = cc.sendLatencyEnable;
|
|||
|
|
++ this.detectInterval = cc.detectInterval;
|
|||
|
|
++ this.detectTimeout = cc.detectTimeout;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public ClientConfig cloneClientConfig() {
|
|||
|
|
+@@ -210,6 +227,10 @@ public class ClientConfig {
|
|||
|
|
+ cc.decodeDecompressBody = decodeDecompressBody;
|
|||
|
|
+ cc.enableStreamRequestType = enableStreamRequestType;
|
|||
|
|
+ cc.useHeartbeatV2 = useHeartbeatV2;
|
|||
|
|
++ cc.startDetectorEnable = startDetectorEnable;
|
|||
|
|
++ cc.sendLatencyEnable = sendLatencyEnable;
|
|||
|
|
++ cc.detectInterval = detectInterval;
|
|||
|
|
++ cc.detectTimeout = detectTimeout;
|
|||
|
|
+ return cc;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -381,6 +402,38 @@ public class ClientConfig {
|
|||
|
|
+ this.enableStreamRequestType = enableStreamRequestType;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public boolean isSendLatencyEnable() {
|
|||
|
|
++ return sendLatencyEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setSendLatencyEnable(boolean sendLatencyEnable) {
|
|||
|
|
++ this.sendLatencyEnable = sendLatencyEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isStartDetectorEnable() {
|
|||
|
|
++ return startDetectorEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setStartDetectorEnable(boolean startDetectorEnable) {
|
|||
|
|
++ this.startDetectorEnable = startDetectorEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public int getDetectTimeout() {
|
|||
|
|
++ return this.detectTimeout;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setDetectTimeout(int detectTimeout) {
|
|||
|
|
++ this.detectTimeout = detectTimeout;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public int getDetectInterval() {
|
|||
|
|
++ return this.detectInterval;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setDetectInterval(int detectInterval) {
|
|||
|
|
++ this.detectInterval = detectInterval;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public boolean isUseHeartbeatV2() {
|
|||
|
|
+ return useHeartbeatV2;
|
|||
|
|
+ }
|
|||
|
|
+@@ -403,6 +456,7 @@ public class ClientConfig {
|
|||
|
|
+ + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name()
|
|||
|
|
+ + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout
|
|||
|
|
+ + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody
|
|||
|
|
++ + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable
|
|||
|
|
+ + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]";
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java
|
|||
|
|
+index 4a3d90135..3a086c13d 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java
|
|||
|
|
+@@ -33,6 +33,14 @@ public class ThreadLocalIndex {
|
|||
|
|
+ return index & POSITIVE_MASK;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public void reset() {
|
|||
|
|
++ int index = Math.abs(random.nextInt(Integer.MAX_VALUE));
|
|||
|
|
++ if (index < 0) {
|
|||
|
|
++ index = 0;
|
|||
|
|
++ }
|
|||
|
|
++ this.threadLocalIndex.set(index);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public String toString() {
|
|||
|
|
+ return "ThreadLocalIndex{" +
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+index 213c26fd6..3201a493f 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+@@ -666,7 +666,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ } catch (Throwable e) {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false);
|
|||
|
|
++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true);
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -684,14 +684,14 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ } catch (Throwable e) {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false);
|
|||
|
|
++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true);
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true);
|
|||
|
|
++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true);
|
|||
|
|
+ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
+ retryTimesWhenSendFailed, times, e, context, false, producer);
|
|||
|
|
+ }
|
|||
|
|
+ } else {
|
|||
|
|
+- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true);
|
|||
|
|
++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true);
|
|||
|
|
+ if (!responseFuture.isSendRequestOK()) {
|
|||
|
|
+ MQClientException ex = new MQClientException("send request failed", responseFuture.getCause());
|
|||
|
|
+ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
+@@ -711,7 +711,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ });
|
|||
|
|
+ } catch (Exception ex) {
|
|||
|
|
+ long cost = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
+- producer.updateFaultItem(brokerName, cost, true);
|
|||
|
|
++ producer.updateFaultItem(brokerName, cost, true, false);
|
|||
|
|
+ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
+ retryTimesWhenSendFailed, times, ex, context, true, producer);
|
|||
|
|
+ }
|
|||
|
|
+@@ -735,7 +735,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ if (needRetry && tmp <= timesTotal) {
|
|||
|
|
+ String retryBrokerName = brokerName;//by default, it will send to the same broker
|
|||
|
|
+ if (topicPublishInfo != null) { //select one message queue accordingly, in order to determine which broker to send
|
|||
|
|
+- MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName);
|
|||
|
|
++ MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName, false);
|
|||
|
|
+ retryBrokerName = instance.getBrokerNameFromMessageQueue(mqChosen);
|
|||
|
|
+ }
|
|||
|
|
+ String addr = instance.findBrokerAddressInPublish(retryBrokerName);
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
|
|||
|
|
+index 8851bc815..9484b26f8 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
|
|||
|
|
+@@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
++import java.util.concurrent.ThreadFactory;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicLong;
|
|||
|
|
+ import java.util.concurrent.locks.Lock;
|
|||
|
|
+@@ -125,6 +126,12 @@ public class MQClientInstance {
|
|||
|
|
+ private final Set<String/* Broker address */> brokerSupportV2HeartbeatSet = new HashSet();
|
|||
|
|
+ private final ConcurrentMap<String, Integer> brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap();
|
|||
|
|
+ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "MQClientFactoryScheduledThread"));
|
|||
|
|
++ private final ScheduledExecutorService fetchRemoteConfigExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public Thread newThread(Runnable r) {
|
|||
|
|
++ return new Thread(r, "MQClientFactoryFetchRemoteConfigScheduledThread");
|
|||
|
|
++ }
|
|||
|
|
++ });
|
|||
|
|
+ private final PullMessageService pullMessageService;
|
|||
|
|
+ private final RebalanceService rebalanceService;
|
|||
|
|
+ private final DefaultMQProducer defaultMQProducer;
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
|
|||
|
|
+index 3f4c6e5f7..bbbb17b07 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
|
|||
|
|
+@@ -33,6 +33,8 @@ import java.util.concurrent.RejectedExecutionException;
|
|||
|
|
+ import java.util.concurrent.Semaphore;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
++
|
|||
|
|
++import com.google.common.base.Optional;
|
|||
|
|
+ import org.apache.rocketmq.client.QueryResult;
|
|||
|
|
+ import org.apache.rocketmq.client.Validators;
|
|||
|
|
+ import org.apache.rocketmq.client.common.ClientErrorCode;
|
|||
|
|
+@@ -49,6 +51,8 @@ import org.apache.rocketmq.client.impl.CommunicationMode;
|
|||
|
|
+ import org.apache.rocketmq.client.impl.MQClientManager;
|
|||
|
|
+ import org.apache.rocketmq.client.impl.factory.MQClientInstance;
|
|||
|
|
+ import org.apache.rocketmq.client.latency.MQFaultStrategy;
|
|||
|
|
++import org.apache.rocketmq.client.latency.Resolver;
|
|||
|
|
++import org.apache.rocketmq.client.latency.ServiceDetector;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.DefaultMQProducer;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.LocalTransactionExecuter;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.LocalTransactionState;
|
|||
|
|
+@@ -112,7 +116,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ private ServiceState serviceState = ServiceState.CREATE_JUST;
|
|||
|
|
+ private MQClientInstance mQClientFactory;
|
|||
|
|
+ private ArrayList<CheckForbiddenHook> checkForbiddenHookList = new ArrayList<>();
|
|||
|
|
+- private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy();
|
|||
|
|
++ private MQFaultStrategy mqFaultStrategy;
|
|||
|
|
+ private ExecutorService asyncSenderExecutor;
|
|||
|
|
+
|
|||
|
|
+ // compression related
|
|||
|
|
+@@ -153,8 +157,38 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ semaphoreAsyncSendSize = new Semaphore(1024 * 1024, true);
|
|||
|
|
+ log.info("semaphoreAsyncSendSize can not be smaller than 1M.");
|
|||
|
|
+ }
|
|||
|
|
+- }
|
|||
|
|
+
|
|||
|
|
++ ServiceDetector serviceDetector = new ServiceDetector() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean detect(String endpoint, long timeoutMillis) {
|
|||
|
|
++ Optional<String> candidateTopic = pickTopic();
|
|||
|
|
++ if (!candidateTopic.isPresent()) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++ try {
|
|||
|
|
++ MessageQueue mq = new MessageQueue(candidateTopic.get(), null, 0);
|
|||
|
|
++ mQClientFactory.getMQClientAPIImpl()
|
|||
|
|
++ .getMaxOffset(endpoint, mq, timeoutMillis);
|
|||
|
|
++ return true;
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ };
|
|||
|
|
++
|
|||
|
|
++ this.mqFaultStrategy = new MQFaultStrategy(defaultMQProducer.cloneClientConfig(), new Resolver() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public String resolve(String name) {
|
|||
|
|
++ return DefaultMQProducerImpl.this.mQClientFactory.findBrokerAddressInPublish(name);
|
|||
|
|
++ }
|
|||
|
|
++ }, serviceDetector);
|
|||
|
|
++ }
|
|||
|
|
++ private Optional<String> pickTopic() {
|
|||
|
|
++ if (topicPublishInfoTable.isEmpty()) {
|
|||
|
|
++ return Optional.absent();
|
|||
|
|
++ }
|
|||
|
|
++ return Optional.of(topicPublishInfoTable.keySet().iterator().next());
|
|||
|
|
++ }
|
|||
|
|
+ public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) {
|
|||
|
|
+ this.checkForbiddenHookList.add(checkForbiddenHook);
|
|||
|
|
+ log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(),
|
|||
|
|
+@@ -229,6 +263,10 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ mQClientFactory.start();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ if (this.mqFaultStrategy.isStartDetectorEnable()) {
|
|||
|
|
++ this.mqFaultStrategy.startDetector();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(),
|
|||
|
|
+ this.defaultMQProducer.isSendMessageWithVIPChannel());
|
|||
|
|
+ this.serviceState = ServiceState.RUNNING;
|
|||
|
|
+@@ -273,6 +311,9 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ if (shutdownFactory) {
|
|||
|
|
+ this.mQClientFactory.shutdown();
|
|||
|
|
+ }
|
|||
|
|
++ if (this.mqFaultStrategy.isStartDetectorEnable()) {
|
|||
|
|
++ this.mqFaultStrategy.shutdown();
|
|||
|
|
++ }
|
|||
|
|
+ RequestFutureHolder.getInstance().shutdown(this);
|
|||
|
|
+ log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup());
|
|||
|
|
+ this.serviceState = ServiceState.SHUTDOWN_ALREADY;
|
|||
|
|
+@@ -574,7 +615,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg,
|
|||
|
|
+- final long timeout) throws MQClientException, RemotingTooMuchRequestException {
|
|||
|
|
++ final long timeout) throws MQClientException, RemotingTooMuchRequestException {
|
|||
|
|
+ long beginStartTime = System.currentTimeMillis();
|
|||
|
|
+ this.makeSureStateOK();
|
|||
|
|
+ Validators.checkMessage(msg, this.defaultMQProducer);
|
|||
|
|
+@@ -584,7 +625,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ MessageQueue mq = null;
|
|||
|
|
+ try {
|
|||
|
|
+ List<MessageQueue> messageQueueList =
|
|||
|
|
+- mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList());
|
|||
|
|
++ mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList());
|
|||
|
|
+ Message userMessage = MessageAccessor.cloneMessage(msg);
|
|||
|
|
+ String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace());
|
|||
|
|
+ userMessage.setTopic(userTopic);
|
|||
|
|
+@@ -609,12 +650,13 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ throw new MQClientException("No route info for this topic, " + msg.getTopic(), null);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) {
|
|||
|
|
+- return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName);
|
|||
|
|
++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) {
|
|||
|
|
++ return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName, resetIndex);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) {
|
|||
|
|
+- this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation);
|
|||
|
|
++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation,
|
|||
|
|
++ boolean reachable) {
|
|||
|
|
++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void validateNameServerSetting() throws MQClientException {
|
|||
|
|
+@@ -647,9 +689,13 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1;
|
|||
|
|
+ int times = 0;
|
|||
|
|
+ String[] brokersSent = new String[timesTotal];
|
|||
|
|
++ boolean resetIndex = false;
|
|||
|
|
+ for (; times < timesTotal; times++) {
|
|||
|
|
+ String lastBrokerName = null == mq ? null : mq.getBrokerName();
|
|||
|
|
+- MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName);
|
|||
|
|
++ if (times > 0) {
|
|||
|
|
++ resetIndex = true;
|
|||
|
|
++ }
|
|||
|
|
++ MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName, resetIndex);
|
|||
|
|
+ if (mqSelected != null) {
|
|||
|
|
+ mq = mqSelected;
|
|||
|
|
+ brokersSent[times] = mq.getBrokerName();
|
|||
|
|
+@@ -667,7 +713,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+
|
|||
|
|
+ sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime);
|
|||
|
|
+ endTimestamp = System.currentTimeMillis();
|
|||
|
|
+- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false);
|
|||
|
|
++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true);
|
|||
|
|
+ switch (communicationMode) {
|
|||
|
|
+ case ASYNC:
|
|||
|
|
+ return null;
|
|||
|
|
+@@ -684,9 +730,22 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ default:
|
|||
|
|
+ break;
|
|||
|
|
+ }
|
|||
|
|
+- } catch (RemotingException | MQClientException e) {
|
|||
|
|
++ } catch (MQClientException e) {
|
|||
|
|
+ endTimestamp = System.currentTimeMillis();
|
|||
|
|
+- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true);
|
|||
|
|
++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true);
|
|||
|
|
++ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e);
|
|||
|
|
++ log.warn(msg.toString());
|
|||
|
|
++ exception = e;
|
|||
|
|
++ continue;
|
|||
|
|
++ } catch (RemotingException e) {
|
|||
|
|
++ endTimestamp = System.currentTimeMillis();
|
|||
|
|
++ if (this.mqFaultStrategy.isStartDetectorEnable()) {
|
|||
|
|
++ // Set this broker unreachable when detecting schedule task is running for RemotingException.
|
|||
|
|
++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false);
|
|||
|
|
++ } else {
|
|||
|
|
++ // Otherwise, isolate this broker.
|
|||
|
|
++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, true);
|
|||
|
|
++ }
|
|||
|
|
+ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e);
|
|||
|
|
+ if (log.isDebugEnabled()) {
|
|||
|
|
+ log.debug(msg.toString());
|
|||
|
|
+@@ -695,7 +754,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ continue;
|
|||
|
|
+ } catch (MQBrokerException e) {
|
|||
|
|
+ endTimestamp = System.currentTimeMillis();
|
|||
|
|
+- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true);
|
|||
|
|
++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false);
|
|||
|
|
+ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e);
|
|||
|
|
+ if (log.isDebugEnabled()) {
|
|||
|
|
+ log.debug(msg.toString());
|
|||
|
|
+@@ -712,7 +771,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ }
|
|||
|
|
+ } catch (InterruptedException e) {
|
|||
|
|
+ endTimestamp = System.currentTimeMillis();
|
|||
|
|
+- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false);
|
|||
|
|
++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true);
|
|||
|
|
+ log.warn("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e);
|
|||
|
|
+ if (log.isDebugEnabled()) {
|
|||
|
|
+ log.debug(msg.toString());
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java
|
|||
|
|
+index 275ada7ac..37b1f3252 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java
|
|||
|
|
+@@ -18,6 +18,8 @@ package org.apache.rocketmq.client.impl.producer;
|
|||
|
|
+
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
++
|
|||
|
|
++import com.google.common.base.Preconditions;
|
|||
|
|
+ import org.apache.rocketmq.client.common.ThreadLocalIndex;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.QueueData;
|
|||
|
|
+@@ -30,6 +32,10 @@ public class TopicPublishInfo {
|
|||
|
|
+ private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex();
|
|||
|
|
+ private TopicRouteData topicRouteData;
|
|||
|
|
+
|
|||
|
|
++ public interface QueueFilter {
|
|||
|
|
++ boolean filter(MessageQueue mq);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public boolean isOrderTopic() {
|
|||
|
|
+ return orderTopic;
|
|||
|
|
+ }
|
|||
|
|
+@@ -66,6 +72,40 @@ public class TopicPublishInfo {
|
|||
|
|
+ this.haveTopicRouterInfo = haveTopicRouterInfo;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public MessageQueue selectOneMessageQueue(QueueFilter ...filter) {
|
|||
|
|
++ return selectOneMessageQueue(this.messageQueueList, this.sendWhichQueue, filter);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private MessageQueue selectOneMessageQueue(List<MessageQueue> messageQueueList, ThreadLocalIndex sendQueue, QueueFilter ...filter) {
|
|||
|
|
++ if (messageQueueList == null || messageQueueList.isEmpty()) {
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ if (filter != null && filter.length != 0) {
|
|||
|
|
++ for (int i = 0; i < messageQueueList.size(); i++) {
|
|||
|
|
++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size());
|
|||
|
|
++ MessageQueue mq = messageQueueList.get(index);
|
|||
|
|
++ boolean filterResult = true;
|
|||
|
|
++ for (QueueFilter f: filter) {
|
|||
|
|
++ Preconditions.checkNotNull(f);
|
|||
|
|
++ filterResult &= f.filter(mq);
|
|||
|
|
++ }
|
|||
|
|
++ if (filterResult) {
|
|||
|
|
++ return mq;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size());
|
|||
|
|
++ return messageQueueList.get(index);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void resetIndex() {
|
|||
|
|
++ this.sendWhichQueue.reset();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public MessageQueue selectOneMessageQueue(final String lastBrokerName) {
|
|||
|
|
+ if (lastBrokerName == null) {
|
|||
|
|
+ return selectOneMessageQueue();
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java
|
|||
|
|
+index 09a8aa461..72d2f3450 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java
|
|||
|
|
+@@ -18,11 +18,75 @@
|
|||
|
|
+ package org.apache.rocketmq.client.latency;
|
|||
|
|
+
|
|||
|
|
+ public interface LatencyFaultTolerance<T> {
|
|||
|
|
+- void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration);
|
|||
|
|
++ /**
|
|||
|
|
++ * Update brokers' states, to decide if they are good or not.
|
|||
|
|
++ *
|
|||
|
|
++ * @param name Broker's name.
|
|||
|
|
++ * @param currentLatency Current message sending process's latency.
|
|||
|
|
++ * @param notAvailableDuration Corresponding not available time, ms. The broker will be not available until it
|
|||
|
|
++ * spends such time.
|
|||
|
|
++ * @param reachable To decide if this broker is reachable or not.
|
|||
|
|
++ */
|
|||
|
|
++ void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration,
|
|||
|
|
++ final boolean reachable);
|
|||
|
|
+
|
|||
|
|
++ /**
|
|||
|
|
++ * To check if this broker is available.
|
|||
|
|
++ *
|
|||
|
|
++ * @param name Broker's name.
|
|||
|
|
++ * @return boolean variable, if this is true, then the broker is available.
|
|||
|
|
++ */
|
|||
|
|
+ boolean isAvailable(final T name);
|
|||
|
|
+
|
|||
|
|
++ /**
|
|||
|
|
++ * To check if this broker is reachable.
|
|||
|
|
++ *
|
|||
|
|
++ * @param name Broker's name.
|
|||
|
|
++ * @return boolean variable, if this is true, then the broker is reachable.
|
|||
|
|
++ */
|
|||
|
|
++ boolean isReachable(final T name);
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * Remove the broker in this fault item table.
|
|||
|
|
++ *
|
|||
|
|
++ * @param name broker's name.
|
|||
|
|
++ */
|
|||
|
|
+ void remove(final T name);
|
|||
|
|
+
|
|||
|
|
++ /**
|
|||
|
|
++ * The worst situation, no broker can be available. Then choose random one.
|
|||
|
|
++ *
|
|||
|
|
++ * @return A random mq will be returned.
|
|||
|
|
++ */
|
|||
|
|
+ T pickOneAtLeast();
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * Start a new thread, to detect the broker's reachable tag.
|
|||
|
|
++ */
|
|||
|
|
++ void startDetector();
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * Shutdown threads that started by LatencyFaultTolerance.
|
|||
|
|
++ */
|
|||
|
|
++ void shutdown();
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * A function reserved, just detect by once, won't create a new thread.
|
|||
|
|
++ */
|
|||
|
|
++ void detectByOneRound();
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * Use it to set the detect timeout bound.
|
|||
|
|
++ *
|
|||
|
|
++ * @param detectTimeout timeout bound
|
|||
|
|
++ */
|
|||
|
|
++ void setDetectTimeout(final int detectTimeout);
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * Use it to set the detector's detector interval for each broker (each broker will be detected once during this
|
|||
|
|
++ * time)
|
|||
|
|
++ *
|
|||
|
|
++ * @param detectInterval each broker's detecting interval
|
|||
|
|
++ */
|
|||
|
|
++ void setDetectInterval(final int detectInterval);
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java
|
|||
|
|
+index 93795d957..8af629574 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java
|
|||
|
|
+@@ -21,30 +21,97 @@ import java.util.Collections;
|
|||
|
|
+ import java.util.Enumeration;
|
|||
|
|
+ import java.util.LinkedList;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
++import java.util.Map;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
++import java.util.concurrent.Executors;
|
|||
|
|
++import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
++import java.util.concurrent.ThreadFactory;
|
|||
|
|
++import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.client.common.ThreadLocalIndex;
|
|||
|
|
++import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+
|
|||
|
|
+ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance<String> {
|
|||
|
|
+- private final ConcurrentHashMap<String, FaultItem> faultItemTable = new ConcurrentHashMap<>(16);
|
|||
|
|
++ private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class);
|
|||
|
|
++ private final ConcurrentHashMap<String, FaultItem> faultItemTable = new ConcurrentHashMap<String, FaultItem>(16);
|
|||
|
|
++ private int detectTimeout = 200;
|
|||
|
|
++ private int detectInterval = 2000;
|
|||
|
|
++ private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex();
|
|||
|
|
++ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public Thread newThread(Runnable r) {
|
|||
|
|
++ return new Thread(r, "LatencyFaultToleranceScheduledThread");
|
|||
|
|
++ }
|
|||
|
|
++ });
|
|||
|
|
+
|
|||
|
|
+- private final ThreadLocalIndex randomItem = new ThreadLocalIndex();
|
|||
|
|
++ private final Resolver resolver;
|
|||
|
|
++
|
|||
|
|
++ private final ServiceDetector serviceDetector;
|
|||
|
|
++
|
|||
|
|
++ public LatencyFaultToleranceImpl(Resolver resolver, ServiceDetector serviceDetector) {
|
|||
|
|
++ this.resolver = resolver;
|
|||
|
|
++ this.serviceDetector = serviceDetector;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void detectByOneRound() {
|
|||
|
|
++ for (Map.Entry<String, FaultItem> item : this.faultItemTable.entrySet()) {
|
|||
|
|
++ FaultItem brokerItem = item.getValue();
|
|||
|
|
++ if (System.currentTimeMillis() - brokerItem.checkStamp >= 0) {
|
|||
|
|
++ brokerItem.checkStamp = System.currentTimeMillis() + this.detectInterval;
|
|||
|
|
++ String brokerAddr = resolver.resolve(brokerItem.getName());
|
|||
|
|
++ if (brokerAddr == null) {
|
|||
|
|
++ faultItemTable.remove(item.getKey());
|
|||
|
|
++ continue;
|
|||
|
|
++ }
|
|||
|
|
++ if (null == serviceDetector) {
|
|||
|
|
++ continue;
|
|||
|
|
++ }
|
|||
|
|
++ boolean serviceOK = serviceDetector.detect(brokerAddr, detectTimeout);
|
|||
|
|
++ if (serviceOK && !brokerItem.reachableFlag) {
|
|||
|
|
++ log.info(brokerItem.name + " is reachable now, then it can be used.");
|
|||
|
|
++ brokerItem.reachableFlag = true;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void startDetector() {
|
|||
|
|
++ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void run() {
|
|||
|
|
++ try {
|
|||
|
|
++ detectByOneRound();
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ log.warn("Unexpected exception raised while detecting service reachability", e);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }, 3, 3, TimeUnit.SECONDS);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void shutdown() {
|
|||
|
|
++ this.scheduledExecutorService.shutdown();
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+- public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration) {
|
|||
|
|
++ public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration,
|
|||
|
|
++ final boolean reachable) {
|
|||
|
|
+ FaultItem old = this.faultItemTable.get(name);
|
|||
|
|
+ if (null == old) {
|
|||
|
|
+ final FaultItem faultItem = new FaultItem(name);
|
|||
|
|
+ faultItem.setCurrentLatency(currentLatency);
|
|||
|
|
+- faultItem.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration);
|
|||
|
|
+-
|
|||
|
|
++ faultItem.updateNotAvailableDuration(notAvailableDuration);
|
|||
|
|
++ faultItem.setReachable(reachable);
|
|||
|
|
+ old = this.faultItemTable.putIfAbsent(name, faultItem);
|
|||
|
|
+- if (old != null) {
|
|||
|
|
+- old.setCurrentLatency(currentLatency);
|
|||
|
|
+- old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration);
|
|||
|
|
+- }
|
|||
|
|
+- } else {
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ if (null != old) {
|
|||
|
|
+ old.setCurrentLatency(currentLatency);
|
|||
|
|
+- old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration);
|
|||
|
|
++ old.updateNotAvailableDuration(notAvailableDuration);
|
|||
|
|
++ old.setReachable(reachable);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ if (!reachable) {
|
|||
|
|
++ log.info(name + " is unreachable, it will not be used until it's reachable");
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -57,6 +124,14 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance<String>
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public boolean isReachable(final String name) {
|
|||
|
|
++ final FaultItem faultItem = this.faultItemTable.get(name);
|
|||
|
|
++ if (faultItem != null) {
|
|||
|
|
++ return faultItem.isReachable();
|
|||
|
|
++ }
|
|||
|
|
++ return true;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public void remove(final String name) {
|
|||
|
|
+ this.faultItemTable.remove(name);
|
|||
|
|
+@@ -65,68 +140,98 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance<String>
|
|||
|
|
+ @Override
|
|||
|
|
+ public String pickOneAtLeast() {
|
|||
|
|
+ final Enumeration<FaultItem> elements = this.faultItemTable.elements();
|
|||
|
|
+- List<FaultItem> tmpList = new LinkedList<>();
|
|||
|
|
++ List<FaultItem> tmpList = new LinkedList<FaultItem>();
|
|||
|
|
+ while (elements.hasMoreElements()) {
|
|||
|
|
+ final FaultItem faultItem = elements.nextElement();
|
|||
|
|
+ tmpList.add(faultItem);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ if (!tmpList.isEmpty()) {
|
|||
|
|
+- Collections.sort(tmpList);
|
|||
|
|
+- final int half = tmpList.size() / 2;
|
|||
|
|
+- if (half <= 0) {
|
|||
|
|
+- return tmpList.get(0).getName();
|
|||
|
|
+- } else {
|
|||
|
|
+- final int i = this.randomItem.incrementAndGet() % half;
|
|||
|
|
+- return tmpList.get(i).getName();
|
|||
|
|
++ Collections.shuffle(tmpList);
|
|||
|
|
++ for (FaultItem faultItem : tmpList) {
|
|||
|
|
++ if (faultItem.reachableFlag) {
|
|||
|
|
++ return faultItem.name;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String toString() {
|
|||
|
|
+ return "LatencyFaultToleranceImpl{" +
|
|||
|
|
+- "faultItemTable=" + faultItemTable +
|
|||
|
|
+- ", whichItemWorst=" + randomItem +
|
|||
|
|
+- '}';
|
|||
|
|
++ "faultItemTable=" + faultItemTable +
|
|||
|
|
++ ", whichItemWorst=" + whichItemWorst +
|
|||
|
|
++ '}';
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setDetectTimeout(final int detectTimeout) {
|
|||
|
|
++ this.detectTimeout = detectTimeout;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- class FaultItem implements Comparable<FaultItem> {
|
|||
|
|
++ public void setDetectInterval(final int detectInterval) {
|
|||
|
|
++ this.detectInterval = detectInterval;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public class FaultItem implements Comparable<FaultItem> {
|
|||
|
|
+ private final String name;
|
|||
|
|
+ private volatile long currentLatency;
|
|||
|
|
+ private volatile long startTimestamp;
|
|||
|
|
++ private volatile long checkStamp;
|
|||
|
|
++ private volatile boolean reachableFlag;
|
|||
|
|
+
|
|||
|
|
+ public FaultItem(final String name) {
|
|||
|
|
+ this.name = name;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public void updateNotAvailableDuration(long notAvailableDuration) {
|
|||
|
|
++ if (notAvailableDuration > 0 && System.currentTimeMillis() + notAvailableDuration > this.startTimestamp) {
|
|||
|
|
++ this.startTimestamp = System.currentTimeMillis() + notAvailableDuration;
|
|||
|
|
++ log.info(name + " will be isolated for " + notAvailableDuration + " ms.");
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public int compareTo(final FaultItem other) {
|
|||
|
|
+ if (this.isAvailable() != other.isAvailable()) {
|
|||
|
|
+- if (this.isAvailable())
|
|||
|
|
++ if (this.isAvailable()) {
|
|||
|
|
+ return -1;
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
+- if (other.isAvailable())
|
|||
|
|
++ if (other.isAvailable()) {
|
|||
|
|
+ return 1;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- if (this.currentLatency < other.currentLatency)
|
|||
|
|
++ if (this.currentLatency < other.currentLatency) {
|
|||
|
|
+ return -1;
|
|||
|
|
+- else if (this.currentLatency > other.currentLatency) {
|
|||
|
|
++ } else if (this.currentLatency > other.currentLatency) {
|
|||
|
|
+ return 1;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- if (this.startTimestamp < other.startTimestamp)
|
|||
|
|
++ if (this.startTimestamp < other.startTimestamp) {
|
|||
|
|
+ return -1;
|
|||
|
|
+- else if (this.startTimestamp > other.startTimestamp) {
|
|||
|
|
++ } else if (this.startTimestamp > other.startTimestamp) {
|
|||
|
|
+ return 1;
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public void setReachable(boolean reachableFlag) {
|
|||
|
|
++ this.reachableFlag = reachableFlag;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setCheckStamp(long checkStamp) {
|
|||
|
|
++ this.checkStamp = checkStamp;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public boolean isAvailable() {
|
|||
|
|
+- return (System.currentTimeMillis() - startTimestamp) >= 0;
|
|||
|
|
++ return reachableFlag && System.currentTimeMillis() >= startTimestamp;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isReachable() {
|
|||
|
|
++ return reachableFlag;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+@@ -139,28 +244,32 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance<String>
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean equals(final Object o) {
|
|||
|
|
+- if (this == o)
|
|||
|
|
++ if (this == o) {
|
|||
|
|
+ return true;
|
|||
|
|
+- if (!(o instanceof FaultItem))
|
|||
|
|
++ }
|
|||
|
|
++ if (!(o instanceof FaultItem)) {
|
|||
|
|
+ return false;
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
+ final FaultItem faultItem = (FaultItem) o;
|
|||
|
|
+
|
|||
|
|
+- if (getCurrentLatency() != faultItem.getCurrentLatency())
|
|||
|
|
++ if (getCurrentLatency() != faultItem.getCurrentLatency()) {
|
|||
|
|
+ return false;
|
|||
|
|
+- if (getStartTimestamp() != faultItem.getStartTimestamp())
|
|||
|
|
++ }
|
|||
|
|
++ if (getStartTimestamp() != faultItem.getStartTimestamp()) {
|
|||
|
|
+ return false;
|
|||
|
|
++ }
|
|||
|
|
+ return getName() != null ? getName().equals(faultItem.getName()) : faultItem.getName() == null;
|
|||
|
|
+-
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String toString() {
|
|||
|
|
+ return "FaultItem{" +
|
|||
|
|
+- "name='" + name + '\'' +
|
|||
|
|
+- ", currentLatency=" + currentLatency +
|
|||
|
|
+- ", startTimestamp=" + startTimestamp +
|
|||
|
|
+- '}';
|
|||
|
|
++ "name='" + name + '\'' +
|
|||
|
|
++ ", currentLatency=" + currentLatency +
|
|||
|
|
++ ", startTimestamp=" + startTimestamp +
|
|||
|
|
++ ", reachableFlag=" + reachableFlag +
|
|||
|
|
++ '}';
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public String getName() {
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java
|
|||
|
|
+index 1e1953fad..c01490784 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java
|
|||
|
|
+@@ -17,25 +17,86 @@
|
|||
|
|
+
|
|||
|
|
+ package org.apache.rocketmq.client.latency;
|
|||
|
|
+
|
|||
|
|
+-import org.apache.commons.lang3.StringUtils;
|
|||
|
|
++import org.apache.rocketmq.client.ClientConfig;
|
|||
|
|
+ import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
|
|||
|
|
++import org.apache.rocketmq.client.impl.producer.TopicPublishInfo.QueueFilter;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+-import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+
|
|||
|
|
+ public class MQFaultStrategy {
|
|||
|
|
+- private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class);
|
|||
|
|
+- private final LatencyFaultTolerance<String> latencyFaultTolerance = new LatencyFaultToleranceImpl();
|
|||
|
|
++ private LatencyFaultTolerance<String> latencyFaultTolerance;
|
|||
|
|
++ private boolean sendLatencyFaultEnable;
|
|||
|
|
++ private boolean startDetectorEnable;
|
|||
|
|
++ private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L};
|
|||
|
|
++ private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L};
|
|||
|
|
+
|
|||
|
|
+- private boolean sendLatencyFaultEnable = false;
|
|||
|
|
++ public static class BrokerFilter implements QueueFilter {
|
|||
|
|
++ private String lastBrokerName;
|
|||
|
|
++
|
|||
|
|
++ public void setLastBrokerName(String lastBrokerName) {
|
|||
|
|
++ this.lastBrokerName = lastBrokerName;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override public boolean filter(MessageQueue mq) {
|
|||
|
|
++ if (lastBrokerName != null) {
|
|||
|
|
++ return !mq.getBrokerName().equals(lastBrokerName);
|
|||
|
|
++ }
|
|||
|
|
++ return true;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private ThreadLocal<BrokerFilter> threadBrokerFilter = new ThreadLocal<BrokerFilter>() {
|
|||
|
|
++ @Override protected BrokerFilter initialValue() {
|
|||
|
|
++ return new BrokerFilter();
|
|||
|
|
++ }
|
|||
|
|
++ };
|
|||
|
|
++
|
|||
|
|
++ private QueueFilter reachableFilter = new QueueFilter() {
|
|||
|
|
++ @Override public boolean filter(MessageQueue mq) {
|
|||
|
|
++ return latencyFaultTolerance.isReachable(mq.getBrokerName());
|
|||
|
|
++ }
|
|||
|
|
++ };
|
|||
|
|
++
|
|||
|
|
++ private QueueFilter availableFilter = new QueueFilter() {
|
|||
|
|
++ @Override public boolean filter(MessageQueue mq) {
|
|||
|
|
++ return latencyFaultTolerance.isAvailable(mq.getBrokerName());
|
|||
|
|
++ }
|
|||
|
|
++ };
|
|||
|
|
++
|
|||
|
|
++
|
|||
|
|
++ public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) {
|
|||
|
|
++ this.setStartDetectorEnable(cc.isStartDetectorEnable());
|
|||
|
|
++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable());
|
|||
|
|
++ this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector);
|
|||
|
|
++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval());
|
|||
|
|
++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ // For unit test.
|
|||
|
|
++ public MQFaultStrategy(ClientConfig cc, LatencyFaultTolerance<String> tolerance) {
|
|||
|
|
++ this.setStartDetectorEnable(cc.isStartDetectorEnable());
|
|||
|
|
++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable());
|
|||
|
|
++ this.latencyFaultTolerance = tolerance;
|
|||
|
|
++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval());
|
|||
|
|
++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout());
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
+- private long[] latencyMax = {50L, 100L, 550L, 1000L, 2000L, 3000L, 15000L};
|
|||
|
|
+- private long[] notAvailableDuration = {0L, 0L, 30000L, 60000L, 120000L, 180000L, 600000L};
|
|||
|
|
+
|
|||
|
|
+ public long[] getNotAvailableDuration() {
|
|||
|
|
+ return notAvailableDuration;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public QueueFilter getAvailableFilter() {
|
|||
|
|
++ return availableFilter;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public QueueFilter getReachableFilter() {
|
|||
|
|
++ return reachableFilter;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public ThreadLocal<BrokerFilter> getThreadBrokerFilter() {
|
|||
|
|
++ return threadBrokerFilter;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public void setNotAvailableDuration(final long[] notAvailableDuration) {
|
|||
|
|
+ this.notAvailableDuration = notAvailableDuration;
|
|||
|
|
+ }
|
|||
|
|
+@@ -56,51 +117,69 @@ public class MQFaultStrategy {
|
|||
|
|
+ this.sendLatencyFaultEnable = sendLatencyFaultEnable;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) {
|
|||
|
|
++ public boolean isStartDetectorEnable() {
|
|||
|
|
++ return startDetectorEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setStartDetectorEnable(boolean startDetectorEnable) {
|
|||
|
|
++ this.startDetectorEnable = startDetectorEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void startDetector() {
|
|||
|
|
++ // user should start the detector
|
|||
|
|
++ // and the thread should not be in running state.
|
|||
|
|
++ if (this.sendLatencyFaultEnable && this.startDetectorEnable) {
|
|||
|
|
++ // start the detector.
|
|||
|
|
++ this.latencyFaultTolerance.startDetector();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void shutdown() {
|
|||
|
|
++ if (this.sendLatencyFaultEnable && this.startDetectorEnable) {
|
|||
|
|
++ this.latencyFaultTolerance.shutdown();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) {
|
|||
|
|
++ BrokerFilter brokerFilter = threadBrokerFilter.get();
|
|||
|
|
++ brokerFilter.setLastBrokerName(lastBrokerName);
|
|||
|
|
+ if (this.sendLatencyFaultEnable) {
|
|||
|
|
+- try {
|
|||
|
|
+- int index = tpInfo.getSendWhichQueue().incrementAndGet();
|
|||
|
|
+- for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) {
|
|||
|
|
+- int pos = index++ % tpInfo.getMessageQueueList().size();
|
|||
|
|
+- MessageQueue mq = tpInfo.getMessageQueueList().get(pos);
|
|||
|
|
+- if (!StringUtils.equals(lastBrokerName, mq.getBrokerName()) && latencyFaultTolerance.isAvailable(mq.getBrokerName())) {
|
|||
|
|
+- return mq;
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- final String notBestBroker = latencyFaultTolerance.pickOneAtLeast();
|
|||
|
|
+- int writeQueueNums = tpInfo.getWriteQueueNumsByBroker(notBestBroker);
|
|||
|
|
+- if (writeQueueNums > 0) {
|
|||
|
|
+- final MessageQueue mq = tpInfo.selectOneMessageQueue();
|
|||
|
|
+- if (notBestBroker != null) {
|
|||
|
|
+- mq.setBrokerName(notBestBroker);
|
|||
|
|
+- mq.setQueueId(tpInfo.getSendWhichQueue().incrementAndGet() % writeQueueNums);
|
|||
|
|
+- }
|
|||
|
|
+- return mq;
|
|||
|
|
+- } else {
|
|||
|
|
+- latencyFaultTolerance.remove(notBestBroker);
|
|||
|
|
+- }
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- log.error("Error occurred when selecting message queue", e);
|
|||
|
|
++ if (resetIndex) {
|
|||
|
|
++ tpInfo.resetIndex();
|
|||
|
|
++ }
|
|||
|
|
++ MessageQueue mq = tpInfo.selectOneMessageQueue(availableFilter, brokerFilter);
|
|||
|
|
++ if (mq != null) {
|
|||
|
|
++ return mq;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ mq = tpInfo.selectOneMessageQueue(reachableFilter, brokerFilter);
|
|||
|
|
++ if (mq != null) {
|
|||
|
|
++ return mq;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return tpInfo.selectOneMessageQueue();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- return tpInfo.selectOneMessageQueue(lastBrokerName);
|
|||
|
|
++ MessageQueue mq = tpInfo.selectOneMessageQueue(brokerFilter);
|
|||
|
|
++ if (mq != null) {
|
|||
|
|
++ return mq;
|
|||
|
|
++ }
|
|||
|
|
++ return tpInfo.selectOneMessageQueue();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) {
|
|||
|
|
++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation,
|
|||
|
|
++ final boolean reachable) {
|
|||
|
|
+ if (this.sendLatencyFaultEnable) {
|
|||
|
|
+- long duration = computeNotAvailableDuration(isolation ? 30000 : currentLatency);
|
|||
|
|
+- this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration);
|
|||
|
|
++ long duration = computeNotAvailableDuration(isolation ? 10000 : currentLatency);
|
|||
|
|
++ this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration, reachable);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private long computeNotAvailableDuration(final long currentLatency) {
|
|||
|
|
+ for (int i = latencyMax.length - 1; i >= 0; i--) {
|
|||
|
|
+- if (currentLatency >= latencyMax[i])
|
|||
|
|
++ if (currentLatency >= latencyMax[i]) {
|
|||
|
|
+ return this.notAvailableDuration[i];
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return 0;
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java
|
|||
|
|
+similarity index 65%
|
|||
|
|
+rename from remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java
|
|||
|
|
+rename to client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java
|
|||
|
|
+index 6aa547047..1c29ba334 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java
|
|||
|
|
+@@ -14,20 +14,9 @@
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+-package org.apache.rocketmq.remoting.protocol.body;
|
|||
|
|
++package org.apache.rocketmq.client.latency;
|
|||
|
|
+
|
|||
|
|
+-import java.util.ArrayList;
|
|||
|
|
+-import java.util.List;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
|
|||
|
|
++public interface Resolver {
|
|||
|
|
+
|
|||
|
|
+-public class GetRemoteClientConfigBody extends RemotingSerializable {
|
|||
|
|
+- private List<String> keys = new ArrayList<>();
|
|||
|
|
+-
|
|||
|
|
+- public List<String> getKeys() {
|
|||
|
|
+- return keys;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- public void setKeys(List<String> keys) {
|
|||
|
|
+- this.keys = keys;
|
|||
|
|
+- }
|
|||
|
|
++ String resolve(String name);
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..c6ffbad1c
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java
|
|||
|
|
+@@ -0,0 +1,30 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++package org.apache.rocketmq.client.latency;
|
|||
|
|
++
|
|||
|
|
++/**
|
|||
|
|
++ * Detect whether the remote service state is normal.
|
|||
|
|
++ */
|
|||
|
|
++public interface ServiceDetector {
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * Check if the remote service is normal.
|
|||
|
|
++ * @param endpoint Service endpoint to check against
|
|||
|
|
++ * @return true if the service is back to normal; false otherwise.
|
|||
|
|
++ */
|
|||
|
|
++ boolean detect(String endpoint, long timeoutMillis);
|
|||
|
|
++}
|
|||
|
|
+diff --git a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java
|
|||
|
|
+index 86690e40b..42ccdae5a 100644
|
|||
|
|
+--- a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java
|
|||
|
|
++++ b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java
|
|||
|
|
+@@ -16,11 +16,14 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.client.latency;
|
|||
|
|
+
|
|||
|
|
+-import java.util.concurrent.TimeUnit;
|
|||
|
|
++import org.awaitility.core.ThrowingRunnable;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
++import java.util.concurrent.TimeUnit;
|
|||
|
|
++
|
|||
|
|
+ import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
++import static org.awaitility.Awaitility.await;
|
|||
|
|
+
|
|||
|
|
+ public class LatencyFaultToleranceImplTest {
|
|||
|
|
+ private LatencyFaultTolerance<String> latencyFaultTolerance;
|
|||
|
|
+@@ -29,28 +32,31 @@ public class LatencyFaultToleranceImplTest {
|
|||
|
|
+
|
|||
|
|
+ @Before
|
|||
|
|
+ public void init() {
|
|||
|
|
+- latencyFaultTolerance = new LatencyFaultToleranceImpl();
|
|||
|
|
++ latencyFaultTolerance = new LatencyFaultToleranceImpl(null, null);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testUpdateFaultItem() throws Exception {
|
|||
|
|
+- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000);
|
|||
|
|
++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true);
|
|||
|
|
+ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse();
|
|||
|
|
+ assertThat(latencyFaultTolerance.isAvailable(anotherBrokerName)).isTrue();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testIsAvailable() throws Exception {
|
|||
|
|
+- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50);
|
|||
|
|
++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50, true);
|
|||
|
|
+ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse();
|
|||
|
|
+
|
|||
|
|
+- TimeUnit.MILLISECONDS.sleep(70);
|
|||
|
|
+- assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue();
|
|||
|
|
++ await().atMost(500, TimeUnit.MILLISECONDS).untilAsserted(new ThrowingRunnable() {
|
|||
|
|
++ @Override public void run() throws Throwable {
|
|||
|
|
++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue();
|
|||
|
|
++ }
|
|||
|
|
++ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testRemove() throws Exception {
|
|||
|
|
+- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000);
|
|||
|
|
++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true);
|
|||
|
|
+ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse();
|
|||
|
|
+ latencyFaultTolerance.remove(brokerName);
|
|||
|
|
+ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue();
|
|||
|
|
+@@ -58,10 +64,20 @@ public class LatencyFaultToleranceImplTest {
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testPickOneAtLeast() throws Exception {
|
|||
|
|
+- latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000);
|
|||
|
|
++ latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true);
|
|||
|
|
+ assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName);
|
|||
|
|
+
|
|||
|
|
+- latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000);
|
|||
|
|
+- assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName);
|
|||
|
|
++ // Bad case, since pickOneAtLeast's behavior becomes random
|
|||
|
|
++ // latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, "127.0.0.1:12011", true);
|
|||
|
|
++ // assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testIsReachable() throws Exception {
|
|||
|
|
++ latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true);
|
|||
|
|
++ assertThat(latencyFaultTolerance.isReachable(brokerName)).isEqualTo(true);
|
|||
|
|
++
|
|||
|
|
++ latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, false);
|
|||
|
|
++ assertThat(latencyFaultTolerance.isReachable(anotherBrokerName)).isEqualTo(false);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+\ No newline at end of file
|
|||
|
|
+diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
|
|||
|
|
+index fada0efd7..485b95c42 100644
|
|||
|
|
+--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
|
|||
|
|
++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
|
|||
|
|
+@@ -41,7 +41,6 @@ import org.apache.rocketmq.remoting.protocol.RequestCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.ResponseCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.GetBrokerMemberGroupResponseBody;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.body.GetRemoteClientConfigBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.RegisterBrokerBody;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.TopicList;
|
|||
|
|
+@@ -132,8 +131,6 @@ public class DefaultRequestProcessor implements NettyRequestProcessor {
|
|||
|
|
+ return this.updateConfig(ctx, request);
|
|||
|
|
+ case RequestCode.GET_NAMESRV_CONFIG:
|
|||
|
|
+ return this.getConfig(ctx, request);
|
|||
|
|
+- case RequestCode.GET_CLIENT_CONFIG:
|
|||
|
|
+- return this.getClientConfigs(ctx, request);
|
|||
|
|
+ default:
|
|||
|
|
+ String error = " request type " + request.getCode() + " not supported";
|
|||
|
|
+ return RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error);
|
|||
|
|
+@@ -661,25 +658,4 @@ public class DefaultRequestProcessor implements NettyRequestProcessor {
|
|||
|
|
+ return response;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private RemotingCommand getClientConfigs(ChannelHandlerContext ctx, RemotingCommand request) {
|
|||
|
|
+- final RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+- final GetRemoteClientConfigBody body = GetRemoteClientConfigBody.decode(request.getBody(), GetRemoteClientConfigBody.class);
|
|||
|
|
+-
|
|||
|
|
+- String content = this.namesrvController.getConfiguration().getClientConfigsFormatString(body.getKeys());
|
|||
|
|
+- if (StringUtils.isNotBlank(content)) {
|
|||
|
|
+- try {
|
|||
|
|
+- response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));
|
|||
|
|
+- } catch (UnsupportedEncodingException e) {
|
|||
|
|
+- log.error("getConfig error, ", e);
|
|||
|
|
+- response.setCode(ResponseCode.SYSTEM_ERROR);
|
|||
|
|
+- response.setRemark("UnsupportedEncodingException " + e);
|
|||
|
|
+- return response;
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+- response.setRemark(null);
|
|||
|
|
+- return response;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+index 2994893d7..b2478fec3 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+@@ -232,6 +232,12 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ private String remotingAccessAddr = "";
|
|||
|
|
+ private int remotingListenPort = 8080;
|
|||
|
|
+
|
|||
|
|
++ // related to proxy's send strategy in cluster mode.
|
|||
|
|
++ private boolean sendLatencyEnable = false;
|
|||
|
|
++ private boolean startDetectorEnable = false;
|
|||
|
|
++ private int detectTimeout = 200;
|
|||
|
|
++ private int detectInterval = 2 * 1000;
|
|||
|
|
++
|
|||
|
|
+ private int remotingHeartbeatThreadPoolNums = 2 * PROCESSOR_NUMBER;
|
|||
|
|
+ private int remotingTopicRouteThreadPoolNums = 2 * PROCESSOR_NUMBER;
|
|||
|
|
+ private int remotingSendMessageThreadPoolNums = 4 * PROCESSOR_NUMBER;
|
|||
|
|
+@@ -1409,6 +1415,46 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public boolean isSendLatencyEnable() {
|
|||
|
|
++ return sendLatencyEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isStartDetectorEnable() {
|
|||
|
|
++ return startDetectorEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setStartDetectorEnable(boolean startDetectorEnable) {
|
|||
|
|
++ this.startDetectorEnable = startDetectorEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setSendLatencyEnable(boolean sendLatencyEnable) {
|
|||
|
|
++ this.sendLatencyEnable = sendLatencyEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public boolean getStartDetectorEnable() {
|
|||
|
|
++ return this.startDetectorEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public boolean getSendLatencyEnable() {
|
|||
|
|
++ return this.sendLatencyEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public int getDetectTimeout() {
|
|||
|
|
++ return detectTimeout;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setDetectTimeout(int detectTimeout) {
|
|||
|
|
++ this.detectTimeout = detectTimeout;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public int getDetectInterval() {
|
|||
|
|
++ return detectInterval;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setDetectInterval(int detectInterval) {
|
|||
|
|
++ this.detectInterval = detectInterval;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public boolean isEnableBatchAck() {
|
|||
|
|
+ return enableBatchAck;
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java
|
|||
|
|
+index 6146c80cd..f670df205 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java
|
|||
|
|
+@@ -382,7 +382,7 @@ public class SendMessageActivity extends AbstractMessingActivity {
|
|||
|
|
+ int bucket = Hashing.consistentHash(shardingKey.hashCode(), writeQueues.size());
|
|||
|
|
+ targetMessageQueue = writeQueues.get(bucket);
|
|||
|
|
+ } else {
|
|||
|
|
+- targetMessageQueue = messageQueueView.getWriteSelector().selectOne(false);
|
|||
|
|
++ targetMessageQueue = messageQueueView.getWriteSelector().selectOneByPipeline(false);
|
|||
|
|
+ }
|
|||
|
|
+ return targetMessageQueue;
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java
|
|||
|
|
+index 0d0c62168..a80f6df0b 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java
|
|||
|
|
+@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.processor;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
++
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.SendResult;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.SendStatus;
|
|||
|
|
+@@ -66,6 +67,8 @@ public class ProducerProcessor extends AbstractProcessor {
|
|||
|
|
+ public CompletableFuture<List<SendResult>> sendMessage(ProxyContext ctx, QueueSelector queueSelector,
|
|||
|
|
+ String producerGroup, int sysFlag, List<Message> messageList, long timeoutMillis) {
|
|||
|
|
+ CompletableFuture<List<SendResult>> future = new CompletableFuture<>();
|
|||
|
|
++ long beginTimestampFirst = System.currentTimeMillis();
|
|||
|
|
++ AddressableMessageQueue messageQueue = null;
|
|||
|
|
+ try {
|
|||
|
|
+ Message message = messageList.get(0);
|
|||
|
|
+ String topic = message.getTopic();
|
|||
|
|
+@@ -79,7 +82,7 @@ public class ProducerProcessor extends AbstractProcessor {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+- AddressableMessageQueue messageQueue = queueSelector.select(ctx,
|
|||
|
|
++ messageQueue = queueSelector.select(ctx,
|
|||
|
|
+ this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(ctx, topic));
|
|||
|
|
+ if (messageQueue == null) {
|
|||
|
|
+ throw new ProxyException(ProxyExceptionCode.FORBIDDEN, "no writable queue");
|
|||
|
|
+@@ -90,6 +93,7 @@ public class ProducerProcessor extends AbstractProcessor {
|
|||
|
|
+ }
|
|||
|
|
+ SendMessageRequestHeader requestHeader = buildSendMessageRequestHeader(messageList, producerGroup, sysFlag, messageQueue.getQueueId());
|
|||
|
|
+
|
|||
|
|
++ AddressableMessageQueue finalMessageQueue = messageQueue;
|
|||
|
|
+ future = this.serviceManager.getMessageService().sendMessage(
|
|||
|
|
+ ctx,
|
|||
|
|
+ messageQueue,
|
|||
|
|
+@@ -102,11 +106,19 @@ public class ProducerProcessor extends AbstractProcessor {
|
|||
|
|
+ if (SendStatus.SEND_OK.equals(sendResult.getSendStatus()) &&
|
|||
|
|
+ tranType == MessageSysFlag.TRANSACTION_PREPARED_TYPE &&
|
|||
|
|
+ StringUtils.isNotBlank(sendResult.getTransactionId())) {
|
|||
|
|
+- fillTransactionData(ctx, producerGroup, messageQueue, sendResult, messageList);
|
|||
|
|
++ fillTransactionData(ctx, producerGroup, finalMessageQueue, sendResult, messageList);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ return sendResultList;
|
|||
|
|
+- }, this.executor);
|
|||
|
|
++ }, this.executor)
|
|||
|
|
++ .whenComplete((result, exception) -> {
|
|||
|
|
++ long endTimestamp = System.currentTimeMillis();
|
|||
|
|
++ if (exception != null) {
|
|||
|
|
++ this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(), endTimestamp - beginTimestampFirst, true, false);
|
|||
|
|
++ } else {
|
|||
|
|
++ this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(),endTimestamp - beginTimestampFirst, false, true);
|
|||
|
|
++ }
|
|||
|
|
++ });
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+ future.completeExceptionally(t);
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java
|
|||
|
|
+index d67b68f38..aced15cee 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java
|
|||
|
|
+@@ -54,7 +54,7 @@ public class LocalTopicRouteService extends TopicRouteService {
|
|||
|
|
+ @Override
|
|||
|
|
+ public MessageQueueView getCurrentMessageQueueView(ProxyContext ctx, String topic) throws Exception {
|
|||
|
|
+ TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic);
|
|||
|
|
+- return new MessageQueueView(topic, toTopicRouteData(topicConfig));
|
|||
|
|
++ return new MessageQueueView(topic, toTopicRouteData(topicConfig), null);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java
|
|||
|
|
+index 85cd18d45..f25fb907e 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java
|
|||
|
|
+@@ -17,6 +17,7 @@
|
|||
|
|
+ package org.apache.rocketmq.proxy.service.route;
|
|||
|
|
+
|
|||
|
|
+ import com.google.common.base.MoreObjects;
|
|||
|
|
++import com.google.common.base.Preconditions;
|
|||
|
|
+ import com.google.common.math.IntMath;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.Collections;
|
|||
|
|
+@@ -30,6 +31,8 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
+ import java.util.stream.Collectors;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
++import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
|
|||
|
|
++import org.apache.rocketmq.client.latency.MQFaultStrategy;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.PermName;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.QueueData;
|
|||
|
|
+@@ -44,8 +47,9 @@ public class MessageQueueSelector {
|
|||
|
|
+ private final Map<String, AddressableMessageQueue> brokerNameQueueMap = new ConcurrentHashMap<>();
|
|||
|
|
+ private final AtomicInteger queueIndex;
|
|||
|
|
+ private final AtomicInteger brokerIndex;
|
|||
|
|
++ private MQFaultStrategy mqFaultStrategy;
|
|||
|
|
+
|
|||
|
|
+- public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, boolean read) {
|
|||
|
|
++ public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, MQFaultStrategy mqFaultStrategy, boolean read) {
|
|||
|
|
+ if (read) {
|
|||
|
|
+ this.queues.addAll(buildRead(topicRouteWrapper));
|
|||
|
|
+ } else {
|
|||
|
|
+@@ -55,6 +59,7 @@ public class MessageQueueSelector {
|
|||
|
|
+ Random random = new Random();
|
|||
|
|
+ this.queueIndex = new AtomicInteger(random.nextInt());
|
|||
|
|
+ this.brokerIndex = new AtomicInteger(random.nextInt());
|
|||
|
|
++ this.mqFaultStrategy = mqFaultStrategy;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private static List<AddressableMessageQueue> buildRead(TopicRouteWrapper topicRoute) {
|
|||
|
|
+@@ -154,6 +159,86 @@ public class MessageQueueSelector {
|
|||
|
|
+ return selectOneByIndex(nextIndex, onlyBroker);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public AddressableMessageQueue selectOneByPipeline(boolean onlyBroker) {
|
|||
|
|
++ if (mqFaultStrategy != null && mqFaultStrategy.isSendLatencyFaultEnable()) {
|
|||
|
|
++ List<MessageQueue> messageQueueList = null;
|
|||
|
|
++ MessageQueue messageQueue = null;
|
|||
|
|
++ if (onlyBroker) {
|
|||
|
|
++ messageQueueList = transferAddressableQueues(brokerActingQueues);
|
|||
|
|
++ } else {
|
|||
|
|
++ messageQueueList = transferAddressableQueues(queues);
|
|||
|
|
++ }
|
|||
|
|
++ AddressableMessageQueue addressableMessageQueue = null;
|
|||
|
|
++
|
|||
|
|
++ // use both available filter.
|
|||
|
|
++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex,
|
|||
|
|
++ mqFaultStrategy.getAvailableFilter(), mqFaultStrategy.getReachableFilter());
|
|||
|
|
++ addressableMessageQueue = transferQueue2Addressable(messageQueue);
|
|||
|
|
++ if (addressableMessageQueue != null) {
|
|||
|
|
++ return addressableMessageQueue;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ // use available filter.
|
|||
|
|
++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex,
|
|||
|
|
++ mqFaultStrategy.getAvailableFilter());
|
|||
|
|
++ addressableMessageQueue = transferQueue2Addressable(messageQueue);
|
|||
|
|
++ if (addressableMessageQueue != null) {
|
|||
|
|
++ return addressableMessageQueue;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ // no available filter, then use reachable filter.
|
|||
|
|
++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex,
|
|||
|
|
++ mqFaultStrategy.getReachableFilter());
|
|||
|
|
++ addressableMessageQueue = transferQueue2Addressable(messageQueue);
|
|||
|
|
++ if (addressableMessageQueue != null) {
|
|||
|
|
++ return addressableMessageQueue;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ // SendLatency is not enabled, or no queue is selected, then select by index.
|
|||
|
|
++ return selectOne(onlyBroker);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private MessageQueue selectOneMessageQueue(List<MessageQueue> messageQueueList, AtomicInteger sendQueue, TopicPublishInfo.QueueFilter...filter) {
|
|||
|
|
++ if (messageQueueList == null || messageQueueList.isEmpty()) {
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++ if (filter != null && filter.length != 0) {
|
|||
|
|
++ for (int i = 0; i < messageQueueList.size(); i++) {
|
|||
|
|
++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size());
|
|||
|
|
++ MessageQueue mq = messageQueueList.get(index);
|
|||
|
|
++ boolean filterResult = true;
|
|||
|
|
++ for (TopicPublishInfo.QueueFilter f: filter) {
|
|||
|
|
++ Preconditions.checkNotNull(f);
|
|||
|
|
++ filterResult &= f.filter(mq);
|
|||
|
|
++ }
|
|||
|
|
++ if (filterResult) {
|
|||
|
|
++ return mq;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public List<MessageQueue> transferAddressableQueues(List<AddressableMessageQueue> addressableMessageQueueList) {
|
|||
|
|
++ if (addressableMessageQueueList == null) {
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ return addressableMessageQueueList.stream()
|
|||
|
|
++ .map(AddressableMessageQueue::getMessageQueue)
|
|||
|
|
++ .collect(Collectors.toList());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private AddressableMessageQueue transferQueue2Addressable(MessageQueue messageQueue) {
|
|||
|
|
++ for (AddressableMessageQueue amq: queues) {
|
|||
|
|
++ if (amq.getMessageQueue().equals(messageQueue)) {
|
|||
|
|
++ return amq;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public AddressableMessageQueue selectNextOne(AddressableMessageQueue last) {
|
|||
|
|
+ boolean onlyBroker = last.getQueueId() < 0;
|
|||
|
|
+ AddressableMessageQueue newOne = last;
|
|||
|
|
+@@ -190,6 +275,14 @@ public class MessageQueueSelector {
|
|||
|
|
+ return brokerActingQueues;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public MQFaultStrategy getMQFaultStrategy() {
|
|||
|
|
++ return mqFaultStrategy;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) {
|
|||
|
|
++ this.mqFaultStrategy = mqFaultStrategy;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean equals(Object o) {
|
|||
|
|
+ if (this == o) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java
|
|||
|
|
+index fe5387cfd..8b3c2f7c8 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java
|
|||
|
|
+@@ -17,20 +17,22 @@
|
|||
|
|
+ package org.apache.rocketmq.proxy.service.route;
|
|||
|
|
+
|
|||
|
|
+ import com.google.common.base.MoreObjects;
|
|||
|
|
++import org.apache.rocketmq.client.latency.MQFaultStrategy;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
|
|||
|
|
+
|
|||
|
|
+ public class MessageQueueView {
|
|||
|
|
+- public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData());
|
|||
|
|
++ public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData(), null);
|
|||
|
|
+
|
|||
|
|
+ private final MessageQueueSelector readSelector;
|
|||
|
|
+ private final MessageQueueSelector writeSelector;
|
|||
|
|
+ private final TopicRouteWrapper topicRouteWrapper;
|
|||
|
|
++ private MQFaultStrategy mqFaultStrategy;
|
|||
|
|
+
|
|||
|
|
+- public MessageQueueView(String topic, TopicRouteData topicRouteData) {
|
|||
|
|
++ public MessageQueueView(String topic, TopicRouteData topicRouteData, MQFaultStrategy mqFaultStrategy) {
|
|||
|
|
+ this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic);
|
|||
|
|
+
|
|||
|
|
+- this.readSelector = new MessageQueueSelector(topicRouteWrapper, true);
|
|||
|
|
+- this.writeSelector = new MessageQueueSelector(topicRouteWrapper, false);
|
|||
|
|
++ this.readSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, true);
|
|||
|
|
++ this.writeSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, false);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public TopicRouteData getTopicRouteData() {
|
|||
|
|
+@@ -65,4 +67,12 @@ public class MessageQueueView {
|
|||
|
|
+ .add("topicRouteWrapper", topicRouteWrapper)
|
|||
|
|
+ .toString();
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public MQFaultStrategy getMQFaultStrategy() {
|
|||
|
|
++ return mqFaultStrategy;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) {
|
|||
|
|
++ this.mqFaultStrategy = mqFaultStrategy;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
+index 84348adc3..74769a423 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
+@@ -25,7 +25,13 @@ import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
++
|
|||
|
|
++import com.google.common.base.Optional;
|
|||
|
|
++import org.apache.rocketmq.client.ClientConfig;
|
|||
|
|
+ import org.apache.rocketmq.client.exception.MQClientException;
|
|||
|
|
++import org.apache.rocketmq.client.latency.MQFaultStrategy;
|
|||
|
|
++import org.apache.rocketmq.client.latency.Resolver;
|
|||
|
|
++import org.apache.rocketmq.client.latency.ServiceDetector;
|
|||
|
|
+ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+@@ -39,6 +45,7 @@ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ProxyConfig;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.ResponseCode;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
|
|||
|
|
+ import org.checkerframework.checker.nullness.qual.NonNull;
|
|||
|
|
+ import org.checkerframework.checker.nullness.qual.Nullable;
|
|||
|
|
+@@ -47,6 +54,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown {
|
|||
|
|
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
+ private final MQClientAPIFactory mqClientAPIFactory;
|
|||
|
|
++ private MQFaultStrategy mqFaultStrategy;
|
|||
|
|
+
|
|||
|
|
+ protected final LoadingCache<String /* topicName */, MessageQueueView> topicCache;
|
|||
|
|
+ protected final ScheduledExecutorService scheduledExecutorService;
|
|||
|
|
+@@ -97,15 +105,83 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+-
|
|||
|
|
++ ServiceDetector serviceDetector = new ServiceDetector() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean detect(String endpoint, long timeoutMillis) {
|
|||
|
|
++ Optional<String> candidateTopic = pickTopic();
|
|||
|
|
++ if (!candidateTopic.isPresent()) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++ try {
|
|||
|
|
++ GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();
|
|||
|
|
++ requestHeader.setTopic(candidateTopic.get());
|
|||
|
|
++ requestHeader.setQueueId(0);
|
|||
|
|
++ Long maxOffset = mqClientAPIFactory.getClient().getMaxOffset(endpoint, requestHeader, timeoutMillis).get();
|
|||
|
|
++ return true;
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ };
|
|||
|
|
++ mqFaultStrategy = new MQFaultStrategy(extractClientConfigFromProxyConfig(config), new Resolver() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public String resolve(String name) {
|
|||
|
|
++ try {
|
|||
|
|
++ String brokerAddr = getBrokerAddr(null, name);
|
|||
|
|
++ return brokerAddr;
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }, serviceDetector);
|
|||
|
|
+ this.init();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ // pickup one topic in the topic cache
|
|||
|
|
++ private Optional<String> pickTopic() {
|
|||
|
|
++ if (topicCache.asMap().isEmpty()) {
|
|||
|
|
++ return Optional.absent();
|
|||
|
|
++ }
|
|||
|
|
++ return Optional.of(topicCache.asMap().keySet().iterator().next());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ protected void init() {
|
|||
|
|
+ this.appendShutdown(this.scheduledExecutorService::shutdown);
|
|||
|
|
+ this.appendStartAndShutdown(this.mqClientAPIFactory);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public void shutdown() throws Exception {
|
|||
|
|
++ if (this.mqFaultStrategy.isStartDetectorEnable()) {
|
|||
|
|
++ mqFaultStrategy.shutdown();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void start() throws Exception {
|
|||
|
|
++ if (this.mqFaultStrategy.isStartDetectorEnable()) {
|
|||
|
|
++ this.mqFaultStrategy.startDetector();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public ClientConfig extractClientConfigFromProxyConfig(ProxyConfig proxyConfig) {
|
|||
|
|
++ ClientConfig tempClientConfig = new ClientConfig();
|
|||
|
|
++ tempClientConfig.setSendLatencyEnable(proxyConfig.getSendLatencyEnable());
|
|||
|
|
++ tempClientConfig.setStartDetectorEnable(proxyConfig.getStartDetectorEnable());
|
|||
|
|
++ tempClientConfig.setDetectTimeout(proxyConfig.getDetectTimeout());
|
|||
|
|
++ tempClientConfig.setDetectInterval(proxyConfig.getDetectInterval());
|
|||
|
|
++ return tempClientConfig;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation,
|
|||
|
|
++ boolean reachable) {
|
|||
|
|
++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public MQFaultStrategy getMqFaultStrategy() {
|
|||
|
|
++ return this.mqFaultStrategy;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public MessageQueueView getAllMessageQueueView(ProxyContext ctx, String topicName) throws Exception {
|
|||
|
|
+ return getCacheMessageQueueWrapper(this.topicCache, topicName);
|
|||
|
|
+ }
|
|||
|
|
+@@ -136,7 +212,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown {
|
|||
|
|
+
|
|||
|
|
+ protected MessageQueueView buildMessageQueueView(String topic, TopicRouteData topicRouteData) {
|
|||
|
|
+ if (isTopicRouteValid(topicRouteData)) {
|
|||
|
|
+- MessageQueueView tmp = new MessageQueueView(topic, topicRouteData);
|
|||
|
|
++ MessageQueueView tmp = new MessageQueueView(topic, topicRouteData, TopicRouteService.this.getMqFaultStrategy());
|
|||
|
|
+ log.debug("load topic route from namesrv. topic: {}, queue: {}", topic, tmp);
|
|||
|
|
+ return tmp;
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java
|
|||
|
|
+index 7fd9a9ffd..77ae5e4d1 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java
|
|||
|
|
+@@ -93,7 +93,6 @@ public class ReceiveMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+ pollTimeCaptor.capture(), anyInt(), any(), anyBoolean(), any(), isNull(), anyLong()))
|
|||
|
|
+ .thenReturn(CompletableFuture.completedFuture(new PopResult(PopStatus.NO_NEW_MSG, Collections.emptyList())));
|
|||
|
|
+
|
|||
|
|
+-
|
|||
|
|
+ ProxyContext context = createContext();
|
|||
|
|
+ context.setRemainingMs(1L);
|
|||
|
|
+ this.receiveMessageActivity.receiveMessage(
|
|||
|
|
+@@ -274,7 +273,7 @@ public class ReceiveMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+- public void testReceiveMessageQueueSelector() {
|
|||
|
|
++ public void testReceiveMessageQueueSelector() throws Exception {
|
|||
|
|
+ TopicRouteData topicRouteData = new TopicRouteData();
|
|||
|
|
+ List<QueueData> queueDatas = new ArrayList<>();
|
|||
|
|
+ for (int i = 0; i < 2; i++) {
|
|||
|
|
+@@ -298,7 +297,7 @@ public class ReceiveMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+ }
|
|||
|
|
+ topicRouteData.setBrokerDatas(brokerDatas);
|
|||
|
|
+
|
|||
|
|
+- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData);
|
|||
|
|
++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null);
|
|||
|
|
+ ReceiveMessageActivity.ReceiveMessageQueueSelector selector = new ReceiveMessageActivity.ReceiveMessageQueueSelector("");
|
|||
|
|
+
|
|||
|
|
+ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView);
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java
|
|||
|
|
+index 588423bb9..4882a5ed8 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java
|
|||
|
|
+@@ -35,6 +35,8 @@ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ExecutionException;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
++import org.apache.rocketmq.client.ClientConfig;
|
|||
|
|
++import org.apache.rocketmq.client.latency.MQFaultStrategy;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.SendResult;
|
|||
|
|
+ import org.apache.rocketmq.client.producer.SendStatus;
|
|||
|
|
+ import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+@@ -49,6 +51,7 @@ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.MessageQueueView;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.route.TopicRouteService;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.BrokerData;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.QueueData;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
|
|||
|
|
+@@ -62,15 +65,19 @@ import static org.junit.Assert.assertThrows;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.any;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyInt;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyString;
|
|||
|
|
++import static org.mockito.Mockito.mock;
|
|||
|
|
+ import static org.mockito.Mockito.when;
|
|||
|
|
+
|
|||
|
|
+ public class SendMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+
|
|||
|
|
+ protected static final String BROKER_NAME = "broker";
|
|||
|
|
++ protected static final String BROKER_NAME2 = "broker2";
|
|||
|
|
+ protected static final String CLUSTER_NAME = "cluster";
|
|||
|
|
+ protected static final String BROKER_ADDR = "127.0.0.1:10911";
|
|||
|
|
++ protected static final String BROKER_ADDR2 = "127.0.0.1:10912";
|
|||
|
|
+ private static final String TOPIC = "topic";
|
|||
|
|
+ private static final String CONSUMER_GROUP = "consumerGroup";
|
|||
|
|
++ MQFaultStrategy mqFaultStrategy;
|
|||
|
|
+
|
|||
|
|
+ private SendMessageActivity sendMessageActivity;
|
|||
|
|
+
|
|||
|
|
+@@ -262,7 +269,7 @@ public class SendMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+- public void testSendOrderMessageQueueSelector() {
|
|||
|
|
++ public void testSendOrderMessageQueueSelector() throws Exception {
|
|||
|
|
+ TopicRouteData topicRouteData = new TopicRouteData();
|
|||
|
|
+ QueueData queueData = new QueueData();
|
|||
|
|
+ BrokerData brokerData = new BrokerData();
|
|||
|
|
+@@ -277,7 +284,7 @@ public class SendMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+ brokerData.setBrokerAddrs(brokerAddrs);
|
|||
|
|
+ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));
|
|||
|
|
+
|
|||
|
|
+- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData);
|
|||
|
|
++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null);
|
|||
|
|
+ SendMessageActivity.SendMessageQueueSelector selector1 = new SendMessageActivity.SendMessageQueueSelector(
|
|||
|
|
+ SendMessageRequest.newBuilder()
|
|||
|
|
+ .addMessages(Message.newBuilder()
|
|||
|
|
+@@ -288,6 +295,12 @@ public class SendMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+ .build()
|
|||
|
|
+ );
|
|||
|
|
+
|
|||
|
|
++ TopicRouteService topicRouteService = mock(TopicRouteService.class);
|
|||
|
|
++ MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class);
|
|||
|
|
++ when(topicRouteService.getAllMessageQueueView(any(), any())).thenReturn(messageQueueView);
|
|||
|
|
++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy);
|
|||
|
|
++ when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false);
|
|||
|
|
++
|
|||
|
|
+ SendMessageActivity.SendMessageQueueSelector selector2 = new SendMessageActivity.SendMessageQueueSelector(
|
|||
|
|
+ SendMessageRequest.newBuilder()
|
|||
|
|
+ .addMessages(Message.newBuilder()
|
|||
|
|
+@@ -328,12 +341,17 @@ public class SendMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+ brokerData.setBrokerAddrs(brokerAddrs);
|
|||
|
|
+ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));
|
|||
|
|
+
|
|||
|
|
+- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData);
|
|||
|
|
++
|
|||
|
|
+ SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector(
|
|||
|
|
+ SendMessageRequest.newBuilder()
|
|||
|
|
+ .addMessages(Message.newBuilder().build())
|
|||
|
|
+ .build()
|
|||
|
|
+ );
|
|||
|
|
++ TopicRouteService topicRouteService = mock(TopicRouteService.class);
|
|||
|
|
++ MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class);
|
|||
|
|
++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy);
|
|||
|
|
++ when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false);
|
|||
|
|
++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy());
|
|||
|
|
+
|
|||
|
|
+ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView);
|
|||
|
|
+ AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView);
|
|||
|
|
+@@ -343,6 +361,45 @@ public class SendMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+ assertNotEquals(firstSelect, secondSelect);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testSendNormalMessageQueueSelectorPipeLine() throws Exception {
|
|||
|
|
++ TopicRouteData topicRouteData = new TopicRouteData();
|
|||
|
|
++ int queueNums = 2;
|
|||
|
|
++
|
|||
|
|
++ QueueData queueData = createQueueData(BROKER_NAME, queueNums);
|
|||
|
|
++ QueueData queueData2 = createQueueData(BROKER_NAME2, queueNums);
|
|||
|
|
++ topicRouteData.setQueueDatas(Lists.newArrayList(queueData,queueData2));
|
|||
|
|
++
|
|||
|
|
++
|
|||
|
|
++ BrokerData brokerData = createBrokerData(CLUSTER_NAME, BROKER_NAME, BROKER_ADDR);
|
|||
|
|
++ BrokerData brokerData2 = createBrokerData(CLUSTER_NAME, BROKER_NAME2, BROKER_ADDR2);
|
|||
|
|
++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData, brokerData2));
|
|||
|
|
++
|
|||
|
|
++ SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector(
|
|||
|
|
++ SendMessageRequest.newBuilder()
|
|||
|
|
++ .addMessages(Message.newBuilder().build())
|
|||
|
|
++ .build()
|
|||
|
|
++ );
|
|||
|
|
++
|
|||
|
|
++ ClientConfig cc = new ClientConfig();
|
|||
|
|
++ this.mqFaultStrategy = new MQFaultStrategy(cc, null, null);
|
|||
|
|
++ mqFaultStrategy.setSendLatencyFaultEnable(true);
|
|||
|
|
++ mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, true);
|
|||
|
|
++ mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, false);
|
|||
|
|
++
|
|||
|
|
++ TopicRouteService topicRouteService = mock(TopicRouteService.class);
|
|||
|
|
++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy);
|
|||
|
|
++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy());
|
|||
|
|
++
|
|||
|
|
++
|
|||
|
|
++ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView);
|
|||
|
|
++ assertEquals(firstSelect.getBrokerName(), BROKER_NAME2);
|
|||
|
|
++
|
|||
|
|
++ mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, false);
|
|||
|
|
++ mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, true);
|
|||
|
|
++ AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView);
|
|||
|
|
++ assertEquals(secondSelect.getBrokerName(), BROKER_NAME);
|
|||
|
|
++ }
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testParameterValidate() {
|
|||
|
|
+ // too large message body
|
|||
|
|
+@@ -850,4 +907,23 @@ public class SendMessageActivityTest extends BaseActivityTest {
|
|||
|
|
+ }
|
|||
|
|
+ return sb.toString();
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ private static QueueData createQueueData(String brokerName, int writeQueueNums) {
|
|||
|
|
++ QueueData queueData = new QueueData();
|
|||
|
|
++ queueData.setBrokerName(brokerName);
|
|||
|
|
++ queueData.setWriteQueueNums(writeQueueNums);
|
|||
|
|
++ queueData.setPerm(PermName.PERM_WRITE);
|
|||
|
|
++ return queueData;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private static BrokerData createBrokerData(String clusterName, String brokerName, String brokerAddrs) {
|
|||
|
|
++ BrokerData brokerData = new BrokerData();
|
|||
|
|
++ brokerData.setCluster(clusterName);
|
|||
|
|
++ brokerData.setBrokerName(brokerName);
|
|||
|
|
++ HashMap<Long, String> brokerAddrsMap = new HashMap<>();
|
|||
|
|
++ brokerAddrsMap.put(MixAll.MASTER_ID, brokerAddrs);
|
|||
|
|
++ brokerData.setBrokerAddrs(brokerAddrsMap);
|
|||
|
|
++
|
|||
|
|
++ return brokerData;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java
|
|||
|
|
+index c97bd5a72..ca6fe909e 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java
|
|||
|
|
+@@ -78,7 +78,7 @@ public class BaseServiceTest extends InitConfigTest {
|
|||
|
|
+ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));
|
|||
|
|
+
|
|||
|
|
+ when(this.topicRouteService.getAllMessageQueueView(any(), eq(ERR_TOPIC))).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, ""));
|
|||
|
|
+- when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData));
|
|||
|
|
+- when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData));
|
|||
|
|
++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null));
|
|||
|
|
++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData, null));
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java
|
|||
|
|
+index e44ed28f4..d150f87c4 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java
|
|||
|
|
+@@ -30,12 +30,12 @@ public class MessageQueueSelectorTest extends BaseServiceTest {
|
|||
|
|
+ public void testReadMessageQueue() {
|
|||
|
|
+ queueData.setPerm(PermName.PERM_READ);
|
|||
|
|
+ queueData.setReadQueueNums(0);
|
|||
|
|
+- MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true);
|
|||
|
|
++ MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true);
|
|||
|
|
+ assertTrue(messageQueueSelector.getQueues().isEmpty());
|
|||
|
|
+
|
|||
|
|
+ queueData.setPerm(PermName.PERM_READ);
|
|||
|
|
+ queueData.setReadQueueNums(3);
|
|||
|
|
+- messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true);
|
|||
|
|
++ messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true);
|
|||
|
|
+ assertEquals(3, messageQueueSelector.getQueues().size());
|
|||
|
|
+ assertEquals(1, messageQueueSelector.getBrokerActingQueues().size());
|
|||
|
|
+ for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) {
|
|||
|
|
+@@ -58,12 +58,12 @@ public class MessageQueueSelectorTest extends BaseServiceTest {
|
|||
|
|
+ public void testWriteMessageQueue() {
|
|||
|
|
+ queueData.setPerm(PermName.PERM_WRITE);
|
|||
|
|
+ queueData.setReadQueueNums(0);
|
|||
|
|
+- MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false);
|
|||
|
|
++ MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false);
|
|||
|
|
+ assertTrue(messageQueueSelector.getQueues().isEmpty());
|
|||
|
|
+
|
|||
|
|
+ queueData.setPerm(PermName.PERM_WRITE);
|
|||
|
|
+ queueData.setWriteQueueNums(3);
|
|||
|
|
+- messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false);
|
|||
|
|
++ messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false);
|
|||
|
|
+ assertEquals(3, messageQueueSelector.getQueues().size());
|
|||
|
|
+ assertEquals(1, messageQueueSelector.getBrokerActingQueues().size());
|
|||
|
|
+ for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) {
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
|
|||
|
|
+index c67f4953d..43fba3d03 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
|
|||
|
|
+@@ -132,7 +132,7 @@ public class HeartbeatSyncerTest extends InitConfigTest {
|
|||
|
|
+ brokerAddr.put(0L, "127.0.0.1:10911");
|
|||
|
|
+ brokerData.setBrokerAddrs(brokerAddr);
|
|||
|
|
+ topicRouteData.getBrokerDatas().add(brokerData);
|
|||
|
|
+- MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData);
|
|||
|
|
++ MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData, null);
|
|||
|
|
+ when(this.topicRouteService.getAllMessageQueueView(any(), anyString())).thenReturn(messageQueueView);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java
|
|||
|
|
+index a0063544e..91af74cbe 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java
|
|||
|
|
+@@ -64,7 +64,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest {
|
|||
|
|
+ this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager,
|
|||
|
|
+ this.mqClientAPIFactory);
|
|||
|
|
+
|
|||
|
|
+- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData);
|
|||
|
|
++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null);
|
|||
|
|
+ when(this.topicRouteService.getAllMessageQueueView(any(), anyString()))
|
|||
|
|
+ .thenReturn(messageQueueView);
|
|||
|
|
+
|
|||
|
|
+@@ -127,7 +127,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest {
|
|||
|
|
+ brokerData.setBrokerAddrs(brokerAddrs);
|
|||
|
|
+ topicRouteData.getQueueDatas().add(queueData);
|
|||
|
|
+ topicRouteData.getBrokerDatas().add(brokerData);
|
|||
|
|
+- when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData));
|
|||
|
|
++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null));
|
|||
|
|
+
|
|||
|
|
+ TopicRouteData clusterTopicRouteData = new TopicRouteData();
|
|||
|
|
+ QueueData clusterQueueData = new QueueData();
|
|||
|
|
+@@ -141,7 +141,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest {
|
|||
|
|
+ brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR);
|
|||
|
|
+ clusterBrokerData.setBrokerAddrs(brokerAddrs);
|
|||
|
|
+ clusterTopicRouteData.setBrokerDatas(Lists.newArrayList(clusterBrokerData));
|
|||
|
|
+- when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData));
|
|||
|
|
++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData, null));
|
|||
|
|
+
|
|||
|
|
+ TopicRouteData clusterTopicRouteData2 = new TopicRouteData();
|
|||
|
|
+ QueueData clusterQueueData2 = new QueueData();
|
|||
|
|
+@@ -155,7 +155,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest {
|
|||
|
|
+ brokerAddrs.put(MixAll.MASTER_ID, brokerAddr2);
|
|||
|
|
+ clusterBrokerData2.setBrokerAddrs(brokerAddrs);
|
|||
|
|
+ clusterTopicRouteData2.setBrokerDatas(Lists.newArrayList(clusterBrokerData2));
|
|||
|
|
+- when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2));
|
|||
|
|
++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2, null));
|
|||
|
|
+
|
|||
|
|
+ ConfigurationManager.getProxyConfig().setTransactionHeartbeatBatchNum(2);
|
|||
|
|
+ this.clusterTransactionService.start();
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch015-backport-fix-some-bugs.patch b/patch015-backport-fix-some-bugs.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..11c10a6bd
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch015-backport-fix-some-bugs.patch
|
|||
|
|
@@ -0,0 +1,1894 @@
|
|||
|
|
+From bd0e9c09db9748f7f74a0c707579142dccf30afc Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: PiteXChen <44110731+RapperCL@users.noreply.github.com>
|
|||
|
|
+Date: Tue, 29 Aug 2023 19:39:27 +0800
|
|||
|
|
+Subject: [PATCH 1/7] [ISSUE #7111] Remove responseFuture from the
|
|||
|
|
+ responseTable when exception occurs (#7112)
|
|||
|
|
+
|
|||
|
|
+* remove responseFuture when exception
|
|||
|
|
+* Empty-Commit
|
|||
|
|
+
|
|||
|
|
+---------
|
|||
|
|
+Co-authored-by: chenyong152 <chenyong152@midea.com>
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/remoting/netty/NettyRemotingAbstract.java | 1 +
|
|||
|
|
+ 1 file changed, 1 insertion(+)
|
|||
|
|
+
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
|
|||
|
|
+index 44d6a3df4..fce2de267 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
|
|||
|
|
+@@ -529,6 +529,7 @@ public abstract class NettyRemotingAbstract {
|
|||
|
|
+ log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel));
|
|||
|
|
+ });
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
++ responseTable.remove(opaque);
|
|||
|
|
+ responseFuture.release();
|
|||
|
|
+ log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e);
|
|||
|
|
+ throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From c78061bf6ca5f35452510ec4107c46735c51c316 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lizhimins <707364882@qq.com>
|
|||
|
|
+Date: Wed, 30 Aug 2023 22:29:51 +0800
|
|||
|
|
+Subject: [PATCH 2/7] [ISSUE#7280] Fix and refactor handle commit exception in
|
|||
|
|
+ tiered storage (#7281)
|
|||
|
|
+
|
|||
|
|
+* refactor handle commit exception
|
|||
|
|
+
|
|||
|
|
+* refactor handle commit exception
|
|||
|
|
+
|
|||
|
|
+* fix handle commit exception
|
|||
|
|
+---
|
|||
|
|
+ .../tieredstore/TieredDispatcher.java | 3 +-
|
|||
|
|
+ .../tieredstore/TieredMessageFetcher.java | 57 ++--
|
|||
|
|
+ .../tieredstore/TieredMessageStore.java | 26 +-
|
|||
|
|
+ .../provider/TieredFileSegment.java | 291 ++++++++++--------
|
|||
|
|
+ .../provider/TieredStoreProvider.java | 8 +-
|
|||
|
|
+ .../provider/posix/PosixFileSegment.java | 4 +-
|
|||
|
|
+ .../CommitLogInputStream.java} | 30 +-
|
|||
|
|
+ .../FileSegmentInputStream.java} | 49 ++-
|
|||
|
|
+ .../FileSegmentInputStreamFactory.java} | 26 +-
|
|||
|
|
+ .../tieredstore/TieredMessageStoreTest.java | 14 +-
|
|||
|
|
+ .../tieredstore/file/TieredFlatFileTest.java | 2 +
|
|||
|
|
+ .../tieredstore/file/TieredIndexFileTest.java | 2 +
|
|||
|
|
+ ...m.java => MockFileSegmentInputStream.java} | 8 +-
|
|||
|
|
+ .../TieredFileSegmentInputStreamTest.java | 24 +-
|
|||
|
|
+ .../provider/TieredFileSegmentTest.java | 89 +++++-
|
|||
|
|
+ .../provider/memory/MemoryFileSegment.java | 27 +-
|
|||
|
|
+ .../memory/MemoryFileSegmentWithoutCheck.java | 4 +-
|
|||
|
|
+ 17 files changed, 427 insertions(+), 237 deletions(-)
|
|||
|
|
+ rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredCommitLogInputStream.java => stream/CommitLogInputStream.java} (88%)
|
|||
|
|
+ rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredFileSegmentInputStream.java => stream/FileSegmentInputStream.java} (77%)
|
|||
|
|
+ rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredFileSegmentInputStreamFactory.java => stream/FileSegmentInputStreamFactory.java} (54%)
|
|||
|
|
+ rename tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/{MockTieredFileSegmentInputStream.java => MockFileSegmentInputStream.java} (82%)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
|
|||
|
|
+index 1746190cd..430c2b62e 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
|
|||
|
|
+@@ -318,8 +318,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch
|
|||
|
|
+ continue;
|
|||
|
|
+ case FILE_CLOSED:
|
|||
|
|
+ tieredFlatFileManager.destroyCompositeFile(flatFile.getMessageQueue());
|
|||
|
|
+- logger.info("TieredDispatcher#dispatchFlatFile: file has been close and destroy, " +
|
|||
|
|
+- "topic: {}, queueId: {}", topic, queueId);
|
|||
|
|
++ logger.info("File has been closed and destroy, topic: {}, queueId: {}", topic, queueId);
|
|||
|
|
+ return;
|
|||
|
|
+ default:
|
|||
|
|
+ dispatchOffset--;
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java
|
|||
|
|
+index 9a9a3e5a5..766ff64f6 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java
|
|||
|
|
+@@ -273,15 +273,17 @@ public class TieredMessageFetcher implements MessageStoreFetcher {
|
|||
|
|
+ TieredStoreMetricsManager.cacheHit.add(resultWrapperList.size(), attributes);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- // if no cached message found and there is currently an inflight request, wait for the request to end before continuing
|
|||
|
|
++ // If there are no messages in the cache and there are currently requests being pulled.
|
|||
|
|
++ // We need to wait for the request to return before continuing.
|
|||
|
|
+ if (resultWrapperList.isEmpty() && waitInflightRequest) {
|
|||
|
|
+- CompletableFuture<Long> future = flatFile.getInflightRequest(group, queueOffset, maxCount)
|
|||
|
|
+- .getFuture(queueOffset);
|
|||
|
|
++ CompletableFuture<Long> future =
|
|||
|
|
++ flatFile.getInflightRequest(group, queueOffset, maxCount).getFuture(queueOffset);
|
|||
|
|
+ if (!future.isDone()) {
|
|||
|
|
+ Stopwatch stopwatch = Stopwatch.createStarted();
|
|||
|
|
+ // to prevent starvation issues, only allow waiting for inflight request once
|
|||
|
|
+ return future.thenCompose(v -> {
|
|||
|
|
+- LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: wait for inflight request cost: {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
|
|||
|
|
++ LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: wait for response cost: {}ms",
|
|||
|
|
++ stopwatch.elapsed(TimeUnit.MILLISECONDS));
|
|||
|
|
+ return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, false);
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+@@ -302,7 +304,8 @@ public class TieredMessageFetcher implements MessageStoreFetcher {
|
|||
|
|
+
|
|||
|
|
+ // if cache hit, result will be returned immediately and asynchronously prefetch messages for later requests
|
|||
|
|
+ if (!resultWrapperList.isEmpty()) {
|
|||
|
|
+- LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: cache hit: topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}",
|
|||
|
|
++ LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: cache hit: " +
|
|||
|
|
++ "topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}",
|
|||
|
|
+ mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, resultWrapperList.size());
|
|||
|
|
+ prefetchMessage(flatFile, group, maxCount, lastGetOffset + 1);
|
|||
|
|
+
|
|||
|
|
+@@ -316,8 +319,10 @@ public class TieredMessageFetcher implements MessageStoreFetcher {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ // if cache is miss, immediately pull messages
|
|||
|
|
+- LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: topic: {}, queue: {}, queue offset: {}, max message num: {}",
|
|||
|
|
++ LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " +
|
|||
|
|
++ "topic: {}, queue: {}, queue offset: {}, max message num: {}",
|
|||
|
|
+ mq.getTopic(), mq.getQueueId(), queueOffset, maxCount);
|
|||
|
|
++
|
|||
|
|
+ CompletableFuture<GetMessageResult> resultFuture;
|
|||
|
|
+ synchronized (flatFile) {
|
|||
|
|
+ int batchSize = maxCount * storeConfig.getReadAheadMinFactor();
|
|||
|
|
+@@ -453,42 +458,42 @@ public class TieredMessageFetcher implements MessageStoreFetcher {
|
|||
|
|
+ public CompletableFuture<GetMessageResult> getMessageAsync(
|
|||
|
|
+ String group, String topic, int queueId, long queueOffset, int maxCount, final MessageFilter messageFilter) {
|
|||
|
|
+
|
|||
|
|
++ GetMessageResult result = new GetMessageResult();
|
|||
|
|
+ CompositeQueueFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId));
|
|||
|
|
++
|
|||
|
|
+ if (flatFile == null) {
|
|||
|
|
+- GetMessageResult result = new GetMessageResult();
|
|||
|
|
+ result.setNextBeginOffset(queueOffset);
|
|||
|
|
+ result.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE);
|
|||
|
|
+ return CompletableFuture.completedFuture(result);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- GetMessageResult result = new GetMessageResult();
|
|||
|
|
+- long minQueueOffset = flatFile.getConsumeQueueMinOffset();
|
|||
|
|
+- long maxQueueOffset = flatFile.getConsumeQueueCommitOffset();
|
|||
|
|
+- result.setMinOffset(minQueueOffset);
|
|||
|
|
+- result.setMaxOffset(maxQueueOffset);
|
|||
|
|
++ // Max queue offset means next message put position
|
|||
|
|
++ result.setMinOffset(flatFile.getConsumeQueueMinOffset());
|
|||
|
|
++ result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());
|
|||
|
|
++
|
|||
|
|
++ // Fill result according file offset.
|
|||
|
|
++ // Offset range | Result | Fix to
|
|||
|
|
++ // (-oo, 0] | no message | current offset
|
|||
|
|
++ // (0, min) | too small | min offset
|
|||
|
|
++ // [min, max) | correct |
|
|||
|
|
++ // [max, max] | overflow one | max offset
|
|||
|
|
++ // (max, +oo) | overflow badly | max offset
|
|||
|
|
+
|
|||
|
|
+- if (flatFile.getConsumeQueueCommitOffset() <= 0) {
|
|||
|
|
++ if (result.getMaxOffset() <= 0) {
|
|||
|
|
+ result.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);
|
|||
|
|
+ result.setNextBeginOffset(queueOffset);
|
|||
|
|
+ return CompletableFuture.completedFuture(result);
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- // request range | result
|
|||
|
|
+- // (0, min) | too small
|
|||
|
|
+- // [min, max) | correct
|
|||
|
|
+- // [max, max] | overflow one
|
|||
|
|
+- // (max, +oo) | overflow badly
|
|||
|
|
+- if (queueOffset < minQueueOffset) {
|
|||
|
|
++ } else if (queueOffset < result.getMinOffset()) {
|
|||
|
|
+ result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL);
|
|||
|
|
+- result.setNextBeginOffset(flatFile.getConsumeQueueMinOffset());
|
|||
|
|
++ result.setNextBeginOffset(result.getMinOffset());
|
|||
|
|
+ return CompletableFuture.completedFuture(result);
|
|||
|
|
+- } else if (queueOffset == maxQueueOffset) {
|
|||
|
|
++ } else if (queueOffset == result.getMaxOffset()) {
|
|||
|
|
+ result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);
|
|||
|
|
+- result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset());
|
|||
|
|
++ result.setNextBeginOffset(result.getMaxOffset());
|
|||
|
|
+ return CompletableFuture.completedFuture(result);
|
|||
|
|
+- } else if (queueOffset > maxQueueOffset) {
|
|||
|
|
++ } else if (queueOffset > result.getMaxOffset()) {
|
|||
|
|
+ result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
|
|||
|
|
+- result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset());
|
|||
|
|
++ result.setNextBeginOffset(result.getMaxOffset());
|
|||
|
|
+ return CompletableFuture.completedFuture(result);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+index 5240ac8e9..78e855f36 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+@@ -99,11 +99,11 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ return storeConfig;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public boolean viaTieredStorage(String topic, int queueId, long offset) {
|
|||
|
|
+- return viaTieredStorage(topic, queueId, offset, 1);
|
|||
|
|
++ public boolean fetchFromCurrentStore(String topic, int queueId, long offset) {
|
|||
|
|
++ return fetchFromCurrentStore(topic, queueId, offset, 1);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public boolean viaTieredStorage(String topic, int queueId, long offset, int batchSize) {
|
|||
|
|
++ public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int batchSize) {
|
|||
|
|
+ TieredMessageStoreConfig.TieredStorageLevel deepStorageLevel = storeConfig.getTieredStorageLevel();
|
|||
|
|
+
|
|||
|
|
+ if (deepStorageLevel.check(TieredMessageStoreConfig.TieredStorageLevel.FORCE)) {
|
|||
|
|
+@@ -146,8 +146,10 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ public CompletableFuture<GetMessageResult> getMessageAsync(String group, String topic,
|
|||
|
|
+ int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) {
|
|||
|
|
+
|
|||
|
|
+- if (!viaTieredStorage(topic, queueId, offset, maxMsgNums)) {
|
|||
|
|
+- logger.trace("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset);
|
|||
|
|
++ if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) {
|
|||
|
|
++ logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset);
|
|||
|
|
++ } else {
|
|||
|
|
++ logger.trace("GetMessageAsync from next store, topic: {}, queue: {}, offset: {}", topic, queueId, offset);
|
|||
|
|
+ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -168,14 +170,14 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+
|
|||
|
|
+ if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) {
|
|||
|
|
+ TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes);
|
|||
|
|
+- logger.debug("GetMessageAsync not found then try back to next store, result: {}, " +
|
|||
|
|
++ logger.debug("GetMessageAsync not found, then back to next store, result: {}, " +
|
|||
|
|
+ "topic: {}, queue: {}, queue offset: {}, offset range: {}-{}",
|
|||
|
|
+ result.getStatus(), topic, queueId, offset, result.getMinOffset(), result.getMaxOffset());
|
|||
|
|
+ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- // system topic
|
|||
|
|
++ // Fetch system topic data from the broker when using the force level.
|
|||
|
|
+ if (result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) {
|
|||
|
|
+ if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) {
|
|||
|
|
+ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);
|
|||
|
|
+@@ -198,7 +200,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ TieredStoreMetricsManager.messagesOutTotal.add(result.getMessageCount(), messagesOutAttributes);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- // fix min or max offset according next store
|
|||
|
|
++ // Fix min or max offset according next store at last
|
|||
|
|
+ long minOffsetInQueue = next.getMinOffsetInQueue(topic, queueId);
|
|||
|
|
+ if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) {
|
|||
|
|
+ result.setMinOffset(minOffsetInQueue);
|
|||
|
|
+@@ -209,7 +211,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ }
|
|||
|
|
+ return result;
|
|||
|
|
+ }).exceptionally(e -> {
|
|||
|
|
+- logger.error("GetMessageAsync from tiered store failed: ", e);
|
|||
|
|
++ logger.error("GetMessageAsync from tiered store failed", e);
|
|||
|
|
+ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+@@ -251,7 +253,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ .build();
|
|||
|
|
+ TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);
|
|||
|
|
+ if (time < 0) {
|
|||
|
|
+- logger.debug("TieredMessageStore#getEarliestMessageTimeAsync: get earliest message time failed, try to get earliest message time from next store: topic: {}, queue: {}",
|
|||
|
|
++ logger.debug("GetEarliestMessageTimeAsync failed, try to get earliest message time from next store: topic: {}, queue: {}",
|
|||
|
|
+ topic, queueId);
|
|||
|
|
+ return finalNextEarliestMessageTime != Long.MAX_VALUE ? finalNextEarliestMessageTime : -1;
|
|||
|
|
+ }
|
|||
|
|
+@@ -262,7 +264,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId,
|
|||
|
|
+ long consumeQueueOffset) {
|
|||
|
|
+- if (viaTieredStorage(topic, queueId, consumeQueueOffset)) {
|
|||
|
|
++ if (fetchFromCurrentStore(topic, queueId, consumeQueueOffset)) {
|
|||
|
|
+ Stopwatch stopwatch = Stopwatch.createStarted();
|
|||
|
|
+ return fetcher.getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset)
|
|||
|
|
+ .thenApply(time -> {
|
|||
|
|
+@@ -272,7 +274,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ .build();
|
|||
|
|
+ TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);
|
|||
|
|
+ if (time == -1) {
|
|||
|
|
+- logger.debug("TieredMessageStore#getMessageStoreTimeStampAsync: get message time failed, try to get message time from next store: topic: {}, queue: {}, queue offset: {}",
|
|||
|
|
++ logger.debug("GetEarliestMessageTimeAsync failed, try to get message time from next store, topic: {}, queue: {}, queue offset: {}",
|
|||
|
|
+ topic, queueId, consumeQueueOffset);
|
|||
|
|
+ return next.getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset);
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java
|
|||
|
|
+index 5062c7d9e..32911a6e8 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java
|
|||
|
|
+@@ -16,14 +16,11 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.tieredstore.provider;
|
|||
|
|
+
|
|||
|
|
+-import com.google.common.annotations.VisibleForTesting;
|
|||
|
|
+-import com.google.common.base.Stopwatch;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.Semaphore;
|
|||
|
|
+-import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.locks.ReentrantLock;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+@@ -35,8 +32,8 @@ import org.apache.rocketmq.tieredstore.exception.TieredStoreException;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.file.TieredCommitLog;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.file.TieredIndexFile;
|
|||
|
|
+-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream;
|
|||
|
|
+-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+
|
|||
|
|
+@@ -50,22 +47,23 @@ public abstract class TieredFileSegment implements Comparable<TieredFileSegment>
|
|||
|
|
+ protected final TieredMessageStoreConfig storeConfig;
|
|||
|
|
+
|
|||
|
|
+ private final long maxSize;
|
|||
|
|
+- private final ReentrantLock bufferLock;
|
|||
|
|
+- private final Semaphore commitLock;
|
|||
|
|
++ private final ReentrantLock bufferLock = new ReentrantLock();
|
|||
|
|
++ private final Semaphore commitLock = new Semaphore(1);
|
|||
|
|
+
|
|||
|
|
+- private volatile boolean full;
|
|||
|
|
+- private volatile boolean closed;
|
|||
|
|
++ private volatile boolean full = false;
|
|||
|
|
++ private volatile boolean closed = false;
|
|||
|
|
+
|
|||
|
|
+- private volatile long minTimestamp;
|
|||
|
|
+- private volatile long maxTimestamp;
|
|||
|
|
+- private volatile long commitPosition;
|
|||
|
|
+- private volatile long appendPosition;
|
|||
|
|
++ private volatile long minTimestamp = Long.MAX_VALUE;
|
|||
|
|
++ private volatile long maxTimestamp = Long.MAX_VALUE;
|
|||
|
|
++ private volatile long commitPosition = 0L;
|
|||
|
|
++ private volatile long appendPosition = 0L;
|
|||
|
|
+
|
|||
|
|
+ // only used in commitLog
|
|||
|
|
+- private volatile long dispatchCommitOffset = 0;
|
|||
|
|
++ private volatile long dispatchCommitOffset = 0L;
|
|||
|
|
+
|
|||
|
|
+ private ByteBuffer codaBuffer;
|
|||
|
|
+- private List<ByteBuffer> uploadBufferList = new ArrayList<>();
|
|||
|
|
++ private List<ByteBuffer> bufferList = new ArrayList<>();
|
|||
|
|
++ private FileSegmentInputStream fileSegmentInputStream;
|
|||
|
|
+ private CompletableFuture<Boolean> flightCommitRequest = CompletableFuture.completedFuture(false);
|
|||
|
|
+
|
|||
|
|
+ public TieredFileSegment(TieredMessageStoreConfig storeConfig,
|
|||
|
|
+@@ -75,21 +73,13 @@ public abstract class TieredFileSegment implements Comparable<TieredFileSegment>
|
|||
|
|
+ this.fileType = fileType;
|
|||
|
|
+ this.filePath = filePath;
|
|||
|
|
+ this.baseOffset = baseOffset;
|
|||
|
|
+-
|
|||
|
|
+- this.closed = false;
|
|||
|
|
+- this.bufferLock = new ReentrantLock();
|
|||
|
|
+- this.commitLock = new Semaphore(1);
|
|||
|
|
+-
|
|||
|
|
+- this.commitPosition = 0L;
|
|||
|
|
+- this.appendPosition = 0L;
|
|||
|
|
+- this.minTimestamp = Long.MAX_VALUE;
|
|||
|
|
+- this.maxTimestamp = Long.MAX_VALUE;
|
|||
|
|
+-
|
|||
|
|
+- // The max segment size of a file is determined by the file type
|
|||
|
|
+- this.maxSize = getMaxSizeAccordingFileType(storeConfig);
|
|||
|
|
++ this.maxSize = getMaxSizeByFileType();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private long getMaxSizeAccordingFileType(TieredMessageStoreConfig storeConfig) {
|
|||
|
|
++ /**
|
|||
|
|
++ * The max segment size of a file is determined by the file type
|
|||
|
|
++ */
|
|||
|
|
++ protected long getMaxSizeByFileType() {
|
|||
|
|
+ switch (fileType) {
|
|||
|
|
+ case COMMIT_LOG:
|
|||
|
|
+ return storeConfig.getTieredStoreCommitLogMaxSize();
|
|||
|
|
+@@ -184,39 +174,23 @@ public abstract class TieredFileSegment implements Comparable<TieredFileSegment>
|
|||
|
|
+ this.appendPosition = pos;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private List<ByteBuffer> rollingUploadBuffer() {
|
|||
|
|
++ private List<ByteBuffer> borrowBuffer() {
|
|||
|
|
+ bufferLock.lock();
|
|||
|
|
+ try {
|
|||
|
|
+- List<ByteBuffer> tmp = uploadBufferList;
|
|||
|
|
+- uploadBufferList = new ArrayList<>();
|
|||
|
|
++ List<ByteBuffer> tmp = bufferList;
|
|||
|
|
++ bufferList = new ArrayList<>();
|
|||
|
|
+ return tmp;
|
|||
|
|
+ } finally {
|
|||
|
|
+ bufferLock.unlock();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private void sendBackBuffer(TieredFileSegmentInputStream inputStream) {
|
|||
|
|
+- bufferLock.lock();
|
|||
|
|
+- try {
|
|||
|
|
+- List<ByteBuffer> tmpBufferList = inputStream.getUploadBufferList();
|
|||
|
|
+- for (ByteBuffer buffer : tmpBufferList) {
|
|||
|
|
+- buffer.rewind();
|
|||
|
|
+- }
|
|||
|
|
+- tmpBufferList.addAll(uploadBufferList);
|
|||
|
|
+- uploadBufferList = tmpBufferList;
|
|||
|
|
+- if (inputStream.getCodaBuffer() != null) {
|
|||
|
|
+- codaBuffer.rewind();
|
|||
|
|
+- }
|
|||
|
|
+- } finally {
|
|||
|
|
+- bufferLock.unlock();
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+ @SuppressWarnings("NonAtomicOperationOnVolatileField")
|
|||
|
|
+- public AppendResult append(ByteBuffer byteBuf, long timeStamp) {
|
|||
|
|
++ public AppendResult append(ByteBuffer byteBuf, long timestamp) {
|
|||
|
|
+ if (closed) {
|
|||
|
|
+ return AppendResult.FILE_CLOSED;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ bufferLock.lock();
|
|||
|
|
+ try {
|
|||
|
|
+ if (full || codaBuffer != null) {
|
|||
|
|
+@@ -227,7 +201,8 @@ public abstract class TieredFileSegment implements Comparable<TieredFileSegment>
|
|||
|
|
+ minTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_BEGIN_TIME_STAMP_POSITION);
|
|||
|
|
+ maxTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_END_TIME_STAMP_POSITION);
|
|||
|
|
+ appendPosition += byteBuf.remaining();
|
|||
|
|
+- uploadBufferList.add(byteBuf);
|
|||
|
|
++ // IndexFile is large and not change after compaction, no need deep copy
|
|||
|
|
++ bufferList.add(byteBuf);
|
|||
|
|
+ setFull();
|
|||
|
|
+ return AppendResult.SUCCESS;
|
|||
|
|
+ }
|
|||
|
|
+@@ -236,23 +211,34 @@ public abstract class TieredFileSegment implements Comparable<TieredFileSegment>
|
|||
|
|
+ setFull();
|
|||
|
|
+ return AppendResult.FILE_FULL;
|
|||
|
|
+ }
|
|||
|
|
+- if (uploadBufferList.size() > storeConfig.getTieredStoreGroupCommitCount()
|
|||
|
|
++
|
|||
|
|
++ if (bufferList.size() > storeConfig.getTieredStoreGroupCommitCount()
|
|||
|
|
+ || appendPosition - commitPosition > storeConfig.getTieredStoreGroupCommitSize()) {
|
|||
|
|
+ commitAsync();
|
|||
|
|
+ }
|
|||
|
|
+- if (uploadBufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) {
|
|||
|
|
+- logger.debug("TieredFileSegment#append: buffer full: file: {}, upload buffer size: {}",
|
|||
|
|
+- getPath(), uploadBufferList.size());
|
|||
|
|
++
|
|||
|
|
++ if (bufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) {
|
|||
|
|
++ logger.debug("File segment append buffer full, file: {}, buffer size: {}, pending bytes: {}",
|
|||
|
|
++ getPath(), bufferList.size(), appendPosition - commitPosition);
|
|||
|
|
+ return AppendResult.BUFFER_FULL;
|
|||
|
|
+ }
|
|||
|
|
+- if (timeStamp != Long.MAX_VALUE) {
|
|||
|
|
+- maxTimestamp = timeStamp;
|
|||
|
|
++
|
|||
|
|
++ if (timestamp != Long.MAX_VALUE) {
|
|||
|
|
++ maxTimestamp = timestamp;
|
|||
|
|
+ if (minTimestamp == Long.MAX_VALUE) {
|
|||
|
|
+- minTimestamp = timeStamp;
|
|||
|
|
++ minTimestamp = timestamp;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ appendPosition += byteBuf.remaining();
|
|||
|
|
+- uploadBufferList.add(byteBuf);
|
|||
|
|
++
|
|||
|
|
++ // deep copy buffer
|
|||
|
|
++ ByteBuffer byteBuffer = ByteBuffer.allocateDirect(byteBuf.remaining());
|
|||
|
|
++ byteBuffer.put(byteBuf);
|
|||
|
|
++ byteBuffer.flip();
|
|||
|
|
++ byteBuf.rewind();
|
|||
|
|
++
|
|||
|
|
++ bufferList.add(byteBuffer);
|
|||
|
|
+ return AppendResult.SUCCESS;
|
|||
|
|
+ } finally {
|
|||
|
|
+ bufferLock.unlock();
|
|||
|
|
+@@ -267,7 +253,6 @@ public abstract class TieredFileSegment implements Comparable<TieredFileSegment>
|
|||
|
|
+ return appendPosition;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- @VisibleForTesting
|
|||
|
|
+ public void setAppendPosition(long appendPosition) {
|
|||
|
|
+ this.appendPosition = appendPosition;
|
|||
|
|
+ }
|
|||
|
|
+@@ -333,6 +318,8 @@ public abstract class TieredFileSegment implements Comparable<TieredFileSegment>
|
|||
|
|
+ if (closed) {
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
++ // result is false when we send real commit request
|
|||
|
|
++ // use join for wait flight request done
|
|||
|
|
+ Boolean result = commitAsync().join();
|
|||
|
|
+ if (!result) {
|
|||
|
|
+ result = flightCommitRequest.join();
|
|||
|
|
+@@ -340,92 +327,156 @@ public abstract class TieredFileSegment implements Comparable<TieredFileSegment>
|
|||
|
|
+ return result;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ private void releaseCommitLock() {
|
|||
|
|
++ if (commitLock.availablePermits() == 0) {
|
|||
|
|
++ commitLock.release();
|
|||
|
|
++ } else {
|
|||
|
|
++ logger.error("[Bug] FileSegmentCommitAsync, lock is already released: available permits: {}",
|
|||
|
|
++ commitLock.availablePermits());
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private void updateDispatchCommitOffset(List<ByteBuffer> bufferList) {
|
|||
|
|
++ if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) {
|
|||
|
|
++ dispatchCommitOffset =
|
|||
|
|
++ MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1));
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * @return false: commit, true: no commit operation
|
|||
|
|
++ */
|
|||
|
|
+ @SuppressWarnings("NonAtomicOperationOnVolatileField")
|
|||
|
|
+ public CompletableFuture<Boolean> commitAsync() {
|
|||
|
|
+ if (closed) {
|
|||
|
|
+ return CompletableFuture.completedFuture(false);
|
|||
|
|
+ }
|
|||
|
|
+- Stopwatch stopwatch = Stopwatch.createStarted();
|
|||
|
|
++
|
|||
|
|
+ if (!needCommit()) {
|
|||
|
|
+ return CompletableFuture.completedFuture(true);
|
|||
|
|
+ }
|
|||
|
|
+- try {
|
|||
|
|
+- int permits = commitLock.drainPermits();
|
|||
|
|
+- if (permits <= 0) {
|
|||
|
|
+- return CompletableFuture.completedFuture(false);
|
|||
|
|
+- }
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
++
|
|||
|
|
++ if (commitLock.drainPermits() <= 0) {
|
|||
|
|
+ return CompletableFuture.completedFuture(false);
|
|||
|
|
+ }
|
|||
|
|
+- List<ByteBuffer> bufferList = rollingUploadBuffer();
|
|||
|
|
+- int bufferSize = 0;
|
|||
|
|
+- for (ByteBuffer buffer : bufferList) {
|
|||
|
|
+- bufferSize += buffer.remaining();
|
|||
|
|
+- }
|
|||
|
|
+- if (codaBuffer != null) {
|
|||
|
|
+- bufferSize += codaBuffer.remaining();
|
|||
|
|
+- }
|
|||
|
|
+- if (bufferSize == 0) {
|
|||
|
|
+- return CompletableFuture.completedFuture(true);
|
|||
|
|
+- }
|
|||
|
|
+- TieredFileSegmentInputStream inputStream = TieredFileSegmentInputStreamFactory.build(
|
|||
|
|
+- fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize);
|
|||
|
|
+- int finalBufferSize = bufferSize;
|
|||
|
|
++
|
|||
|
|
+ try {
|
|||
|
|
+- flightCommitRequest = commit0(inputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX)
|
|||
|
|
++ if (fileSegmentInputStream != null) {
|
|||
|
|
++ long fileSize = this.getSize();
|
|||
|
|
++ if (fileSize == -1L) {
|
|||
|
|
++ logger.error("Get commit position error before commit, Commit: %d, Expect: %d, Current Max: %d, FileName: %s",
|
|||
|
|
++ commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath());
|
|||
|
|
++ releaseCommitLock();
|
|||
|
|
++ return CompletableFuture.completedFuture(false);
|
|||
|
|
++ } else {
|
|||
|
|
++ if (correctPosition(fileSize, null)) {
|
|||
|
|
++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList());
|
|||
|
|
++ fileSegmentInputStream = null;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ int bufferSize;
|
|||
|
|
++ if (fileSegmentInputStream != null) {
|
|||
|
|
++ bufferSize = fileSegmentInputStream.available();
|
|||
|
|
++ } else {
|
|||
|
|
++ List<ByteBuffer> bufferList = borrowBuffer();
|
|||
|
|
++ bufferSize = bufferList.stream().mapToInt(ByteBuffer::remaining).sum()
|
|||
|
|
++ + (codaBuffer != null ? codaBuffer.remaining() : 0);
|
|||
|
|
++ if (bufferSize == 0) {
|
|||
|
|
++ releaseCommitLock();
|
|||
|
|
++ return CompletableFuture.completedFuture(true);
|
|||
|
|
++ }
|
|||
|
|
++ fileSegmentInputStream = FileSegmentInputStreamFactory.build(
|
|||
|
|
++ fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ return flightCommitRequest = this
|
|||
|
|
++ .commit0(fileSegmentInputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX)
|
|||
|
|
+ .thenApply(result -> {
|
|||
|
|
+ if (result) {
|
|||
|
|
+- if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) {
|
|||
|
|
+- dispatchCommitOffset = MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1));
|
|||
|
|
+- }
|
|||
|
|
+- commitPosition += finalBufferSize;
|
|||
|
|
++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList());
|
|||
|
|
++ commitPosition += bufferSize;
|
|||
|
|
++ fileSegmentInputStream = null;
|
|||
|
|
+ return true;
|
|||
|
|
+- }
|
|||
|
|
+- sendBackBuffer(inputStream);
|
|||
|
|
+- return false;
|
|||
|
|
+- })
|
|||
|
|
+- .exceptionally(e -> handleCommitException(inputStream, e))
|
|||
|
|
+- .whenComplete((result, e) -> {
|
|||
|
|
+- if (commitLock.availablePermits() == 0) {
|
|||
|
|
+- logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize);
|
|||
|
|
+- commitLock.release();
|
|||
|
|
+ } else {
|
|||
|
|
+- logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits());
|
|||
|
|
++ fileSegmentInputStream.rewind();
|
|||
|
|
++ return false;
|
|||
|
|
+ }
|
|||
|
|
+- });
|
|||
|
|
+- return flightCommitRequest;
|
|||
|
|
++ })
|
|||
|
|
++ .exceptionally(this::handleCommitException)
|
|||
|
|
++ .whenComplete((result, e) -> releaseCommitLock());
|
|||
|
|
++
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- handleCommitException(inputStream, e);
|
|||
|
|
+- if (commitLock.availablePermits() == 0) {
|
|||
|
|
+- logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize);
|
|||
|
|
+- commitLock.release();
|
|||
|
|
+- } else {
|
|||
|
|
+- logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits());
|
|||
|
|
+- }
|
|||
|
|
++ handleCommitException(e);
|
|||
|
|
++ releaseCommitLock();
|
|||
|
|
+ }
|
|||
|
|
+ return CompletableFuture.completedFuture(false);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private boolean handleCommitException(TieredFileSegmentInputStream inputStream, Throwable e) {
|
|||
|
|
++ private long getCorrectFileSize(Throwable throwable) {
|
|||
|
|
++ if (throwable instanceof TieredStoreException) {
|
|||
|
|
++ long fileSize = ((TieredStoreException) throwable).getPosition();
|
|||
|
|
++ if (fileSize > 0) {
|
|||
|
|
++ return fileSize;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ return getSize();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private boolean handleCommitException(Throwable e) {
|
|||
|
|
++ // Get root cause here
|
|||
|
|
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
|
|||
|
|
+- sendBackBuffer(inputStream);
|
|||
|
|
+- long realSize = 0;
|
|||
|
|
+- if (cause instanceof TieredStoreException && ((TieredStoreException) cause).getPosition() > 0) {
|
|||
|
|
+- realSize = ((TieredStoreException) cause).getPosition();
|
|||
|
|
++ long fileSize = this.getCorrectFileSize(cause);
|
|||
|
|
++
|
|||
|
|
++ if (fileSize == -1L) {
|
|||
|
|
++ logger.error("Get commit position error, Commit: %d, Expect: %d, Current Max: %d, FileName: %s",
|
|||
|
|
++ commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath());
|
|||
|
|
++ fileSegmentInputStream.rewind();
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ if (correctPosition(fileSize, cause)) {
|
|||
|
|
++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList());
|
|||
|
|
++ fileSegmentInputStream = null;
|
|||
|
|
++ return true;
|
|||
|
|
++ } else {
|
|||
|
|
++ fileSegmentInputStream.rewind();
|
|||
|
|
++ return false;
|
|||
|
|
+ }
|
|||
|
|
+- if (realSize <= 0) {
|
|||
|
|
+- realSize = getSize();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * return true to clear buffer
|
|||
|
|
++ */
|
|||
|
|
++ private boolean correctPosition(long fileSize, Throwable throwable) {
|
|||
|
|
++
|
|||
|
|
++ // Current we have three offsets here: commit offset, expect offset, file size.
|
|||
|
|
++ // We guarantee that the commit offset is less than or equal to the expect offset.
|
|||
|
|
++ // Max offset will increase because we can continuously put in new buffers
|
|||
|
|
++ String handleInfo = throwable == null ? "before commit" : "after commit";
|
|||
|
|
++ long expectPosition = commitPosition + fileSegmentInputStream.getContentLength();
|
|||
|
|
++
|
|||
|
|
++ String offsetInfo = String.format("Correct Commit Position, %s, result=[{}], " +
|
|||
|
|
++ "Commit: %d, Expect: %d, Current Max: %d, FileSize: %d, FileName: %s",
|
|||
|
|
++ handleInfo, commitPosition, expectPosition, appendPosition, fileSize, this.getPath());
|
|||
|
|
++
|
|||
|
|
++ // We are believing that the file size returned by the server is correct,
|
|||
|
|
++ // can reset the commit offset to the file size reported by the storage system.
|
|||
|
|
++ if (fileSize == expectPosition) {
|
|||
|
|
++ logger.info(offsetInfo, "Success", throwable);
|
|||
|
|
++ commitPosition = fileSize;
|
|||
|
|
++ return true;
|
|||
|
|
+ }
|
|||
|
|
+- if (realSize > 0 && realSize > commitPosition) {
|
|||
|
|
+- logger.error("TieredFileSegment#handleCommitException: commit failed: file: {}, try to fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause);
|
|||
|
|
+- // TODO check if this diff part is uploaded to backend storage
|
|||
|
|
+- long diff = appendPosition - commitPosition;
|
|||
|
|
+- commitPosition = realSize;
|
|||
|
|
+- appendPosition = realSize + diff;
|
|||
|
|
+- // TODO check if appendPosition is large than maxOffset
|
|||
|
|
+- } else if (realSize < commitPosition) {
|
|||
|
|
+- logger.error("[Bug]TieredFileSegment#handleCommitException: commit failed: file: {}, can not fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause);
|
|||
|
|
++
|
|||
|
|
++ if (fileSize < commitPosition) {
|
|||
|
|
++ logger.error(offsetInfo, "FileSizeIncorrect", throwable);
|
|||
|
|
++ } else if (fileSize == commitPosition) {
|
|||
|
|
++ logger.warn(offsetInfo, "CommitFailed", throwable);
|
|||
|
|
++ } else if (fileSize > commitPosition) {
|
|||
|
|
++ logger.warn(offsetInfo, "PartialSuccess", throwable);
|
|||
|
|
+ }
|
|||
|
|
++ commitPosition = fileSize;
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java
|
|||
|
|
+index 5a0ca25f5..0db3eaf8f 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java
|
|||
|
|
+@@ -18,7 +18,7 @@ package org.apache.rocketmq.tieredstore.provider;
|
|||
|
|
+
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
|
|||
|
|
+
|
|||
|
|
+ public interface TieredStoreProvider {
|
|||
|
|
+
|
|||
|
|
+@@ -30,7 +30,9 @@ public interface TieredStoreProvider {
|
|||
|
|
+ String getPath();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+- * Get file size in backend file system
|
|||
|
|
++ * Get the real length of the file.
|
|||
|
|
++ * Return 0 if the file does not exist,
|
|||
|
|
++ * Return -1 if system get size failed.
|
|||
|
|
+ *
|
|||
|
|
+ * @return file real size
|
|||
|
|
+ */
|
|||
|
|
+@@ -71,5 +73,5 @@ public interface TieredStoreProvider {
|
|||
|
|
+ * @param append try to append or create a new file
|
|||
|
|
+ * @return put result, <code>true</code> if data successfully write; <code>false</code> otherwise
|
|||
|
|
+ */
|
|||
|
|
+- CompletableFuture<Boolean> commit0(TieredFileSegmentInputStream inputStream,long position, int length, boolean append);
|
|||
|
|
++ CompletableFuture<Boolean> commit0(FileSegmentInputStream inputStream,long position, int length, boolean append);
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java
|
|||
|
|
+index 52be90b1d..7e949cb28 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java
|
|||
|
|
+@@ -36,7 +36,7 @@ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
|
|||
|
|
+-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+
|
|||
|
|
+ import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_FILE_TYPE;
|
|||
|
|
+@@ -184,7 +184,7 @@ public class PosixFileSegment extends TieredFileSegment {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<Boolean> commit0(
|
|||
|
|
+- TieredFileSegmentInputStream inputStream, long position, int length, boolean append) {
|
|||
|
|
++ FileSegmentInputStream inputStream, long position, int length, boolean append) {
|
|||
|
|
+
|
|||
|
|
+ Stopwatch stopwatch = Stopwatch.createStarted();
|
|||
|
|
+ AttributesBuilder attributesBuilder = newAttributesBuilder()
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java
|
|||
|
|
+similarity index 88%
|
|||
|
|
+rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java
|
|||
|
|
+rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java
|
|||
|
|
+index c70bb7656..13b6e0ef9 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java
|
|||
|
|
+@@ -15,7 +15,7 @@
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+
|
|||
|
|
+-package org.apache.rocketmq.tieredstore.provider.inputstream;
|
|||
|
|
++package org.apache.rocketmq.tieredstore.provider.stream;
|
|||
|
|
+
|
|||
|
|
+ import java.io.IOException;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+@@ -23,20 +23,23 @@ import java.util.List;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.FileSegmentType;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
|
|||
|
|
+
|
|||
|
|
+-public class TieredCommitLogInputStream extends TieredFileSegmentInputStream {
|
|||
|
|
++public class CommitLogInputStream extends FileSegmentInputStream {
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * commitLogOffset is the real physical offset of the commitLog buffer which is being read
|
|||
|
|
+ */
|
|||
|
|
++ private final long startCommitLogOffset;
|
|||
|
|
++
|
|||
|
|
+ private long commitLogOffset;
|
|||
|
|
+
|
|||
|
|
+ private final ByteBuffer codaBuffer;
|
|||
|
|
+
|
|||
|
|
+ private long markCommitLogOffset = -1;
|
|||
|
|
+
|
|||
|
|
+- public TieredCommitLogInputStream(FileSegmentType fileType, long startOffset,
|
|||
|
|
++ public CommitLogInputStream(FileSegmentType fileType, long startOffset,
|
|||
|
|
+ List<ByteBuffer> uploadBufferList, ByteBuffer codaBuffer, int contentLength) {
|
|||
|
|
+ super(fileType, uploadBufferList, contentLength);
|
|||
|
|
++ this.startCommitLogOffset = startOffset;
|
|||
|
|
+ this.commitLogOffset = startOffset;
|
|||
|
|
+ this.codaBuffer = codaBuffer;
|
|||
|
|
+ }
|
|||
|
|
+@@ -53,6 +56,15 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream {
|
|||
|
|
+ this.commitLogOffset = markCommitLogOffset;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public synchronized void rewind() {
|
|||
|
|
++ super.rewind();
|
|||
|
|
++ this.commitLogOffset = this.startCommitLogOffset;
|
|||
|
|
++ if (this.codaBuffer != null) {
|
|||
|
|
++ this.codaBuffer.rewind();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public ByteBuffer getCodaBuffer() {
|
|||
|
|
+ return this.codaBuffer;
|
|||
|
|
+@@ -64,17 +76,17 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream {
|
|||
|
|
+ return -1;
|
|||
|
|
+ }
|
|||
|
|
+ readPosition++;
|
|||
|
|
+- if (curReadBufferIndex >= uploadBufferList.size()) {
|
|||
|
|
++ if (curReadBufferIndex >= bufferList.size()) {
|
|||
|
|
+ return readCoda();
|
|||
|
|
+ }
|
|||
|
|
+ int res;
|
|||
|
|
+ if (readPosInCurBuffer >= curBuffer.remaining()) {
|
|||
|
|
+ curReadBufferIndex++;
|
|||
|
|
+- if (curReadBufferIndex >= uploadBufferList.size()) {
|
|||
|
|
++ if (curReadBufferIndex >= bufferList.size()) {
|
|||
|
|
+ readPosInCurBuffer = 0;
|
|||
|
|
+ return readCoda();
|
|||
|
|
+ }
|
|||
|
|
+- curBuffer = uploadBufferList.get(curReadBufferIndex);
|
|||
|
|
++ curBuffer = bufferList.get(curReadBufferIndex);
|
|||
|
|
+ commitLogOffset += readPosInCurBuffer;
|
|||
|
|
+ readPosInCurBuffer = 0;
|
|||
|
|
+ }
|
|||
|
|
+@@ -119,9 +131,9 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream {
|
|||
|
|
+ int posInCurBuffer = readPosInCurBuffer;
|
|||
|
|
+ long curCommitLogOffset = commitLogOffset;
|
|||
|
|
+ ByteBuffer curBuf = curBuffer;
|
|||
|
|
+- while (needRead > 0 && bufIndex <= uploadBufferList.size()) {
|
|||
|
|
++ while (needRead > 0 && bufIndex <= bufferList.size()) {
|
|||
|
|
+ int readLen, remaining, realReadLen = 0;
|
|||
|
|
+- if (bufIndex == uploadBufferList.size()) {
|
|||
|
|
++ if (bufIndex == bufferList.size()) {
|
|||
|
|
+ // read from coda buffer
|
|||
|
|
+ remaining = codaBuffer.remaining() - posInCurBuffer;
|
|||
|
|
+ readLen = Math.min(remaining, needRead);
|
|||
|
|
+@@ -137,7 +149,7 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream {
|
|||
|
|
+ }
|
|||
|
|
+ remaining = curBuf.remaining() - posInCurBuffer;
|
|||
|
|
+ readLen = Math.min(remaining, needRead);
|
|||
|
|
+- curBuf = uploadBufferList.get(bufIndex);
|
|||
|
|
++ curBuf = bufferList.get(bufIndex);
|
|||
|
|
+ if (posInCurBuffer < MessageBufferUtil.PHYSICAL_OFFSET_POSITION) {
|
|||
|
|
+ realReadLen = Math.min(MessageBufferUtil.PHYSICAL_OFFSET_POSITION - posInCurBuffer, readLen);
|
|||
|
|
+ // read from commitLog buffer
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java
|
|||
|
|
+similarity index 77%
|
|||
|
|
+rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java
|
|||
|
|
+rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java
|
|||
|
|
+index e1758ca93..9e9d5135c 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java
|
|||
|
|
+@@ -15,15 +15,16 @@
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+
|
|||
|
|
+-package org.apache.rocketmq.tieredstore.provider.inputstream;
|
|||
|
|
++package org.apache.rocketmq.tieredstore.provider.stream;
|
|||
|
|
+
|
|||
|
|
+ import java.io.IOException;
|
|||
|
|
+ import java.io.InputStream;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
++import org.apache.commons.collections.CollectionUtils;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.FileSegmentType;
|
|||
|
|
+
|
|||
|
|
+-public class TieredFileSegmentInputStream extends InputStream {
|
|||
|
|
++public class FileSegmentInputStream extends InputStream {
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * file type, can be commitlog, consume queue or indexfile now
|
|||
|
|
+@@ -33,7 +34,7 @@ public class TieredFileSegmentInputStream extends InputStream {
|
|||
|
|
+ /**
|
|||
|
|
+ * hold bytebuffer
|
|||
|
|
+ */
|
|||
|
|
+- protected final List<ByteBuffer> uploadBufferList;
|
|||
|
|
++ protected final List<ByteBuffer> bufferList;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * total remaining of bytebuffer list
|
|||
|
|
+@@ -65,13 +66,13 @@ public class TieredFileSegmentInputStream extends InputStream {
|
|||
|
|
+
|
|||
|
|
+ private int markReadPosInCurBuffer = -1;
|
|||
|
|
+
|
|||
|
|
+- public TieredFileSegmentInputStream(FileSegmentType fileType, List<ByteBuffer> uploadBufferList,
|
|||
|
|
+- int contentLength) {
|
|||
|
|
++ public FileSegmentInputStream(
|
|||
|
|
++ FileSegmentType fileType, List<ByteBuffer> bufferList, int contentLength) {
|
|||
|
|
+ this.fileType = fileType;
|
|||
|
|
+ this.contentLength = contentLength;
|
|||
|
|
+- this.uploadBufferList = uploadBufferList;
|
|||
|
|
+- if (uploadBufferList != null && uploadBufferList.size() > 0) {
|
|||
|
|
+- this.curBuffer = uploadBufferList.get(curReadBufferIndex);
|
|||
|
|
++ this.bufferList = bufferList;
|
|||
|
|
++ if (bufferList != null && bufferList.size() > 0) {
|
|||
|
|
++ this.curBuffer = bufferList.get(curReadBufferIndex);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -95,18 +96,34 @@ public class TieredFileSegmentInputStream extends InputStream {
|
|||
|
|
+ this.readPosition = markReadPosition;
|
|||
|
|
+ this.curReadBufferIndex = markCurReadBufferIndex;
|
|||
|
|
+ this.readPosInCurBuffer = markReadPosInCurBuffer;
|
|||
|
|
+- if (this.curReadBufferIndex < uploadBufferList.size()) {
|
|||
|
|
+- this.curBuffer = uploadBufferList.get(curReadBufferIndex);
|
|||
|
|
++ if (this.curReadBufferIndex < bufferList.size()) {
|
|||
|
|
++ this.curBuffer = bufferList.get(curReadBufferIndex);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public synchronized void rewind() {
|
|||
|
|
++ this.readPosition = 0;
|
|||
|
|
++ this.curReadBufferIndex = 0;
|
|||
|
|
++ this.readPosInCurBuffer = 0;
|
|||
|
|
++ if (CollectionUtils.isNotEmpty(bufferList)) {
|
|||
|
|
++ this.curBuffer = bufferList.get(0);
|
|||
|
|
++ for (ByteBuffer buffer : bufferList) {
|
|||
|
|
++ buffer.rewind();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public int getContentLength() {
|
|||
|
|
++ return contentLength;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public int available() {
|
|||
|
|
+ return contentLength - readPosition;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public List<ByteBuffer> getUploadBufferList() {
|
|||
|
|
+- return uploadBufferList;
|
|||
|
|
++ public List<ByteBuffer> getBufferList() {
|
|||
|
|
++ return bufferList;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public ByteBuffer getCodaBuffer() {
|
|||
|
|
+@@ -121,10 +138,10 @@ public class TieredFileSegmentInputStream extends InputStream {
|
|||
|
|
+ readPosition++;
|
|||
|
|
+ if (readPosInCurBuffer >= curBuffer.remaining()) {
|
|||
|
|
+ curReadBufferIndex++;
|
|||
|
|
+- if (curReadBufferIndex >= uploadBufferList.size()) {
|
|||
|
|
++ if (curReadBufferIndex >= bufferList.size()) {
|
|||
|
|
+ return -1;
|
|||
|
|
+ }
|
|||
|
|
+- curBuffer = uploadBufferList.get(curReadBufferIndex);
|
|||
|
|
++ curBuffer = bufferList.get(curReadBufferIndex);
|
|||
|
|
+ readPosInCurBuffer = 0;
|
|||
|
|
+ }
|
|||
|
|
+ return curBuffer.get(readPosInCurBuffer++) & 0xff;
|
|||
|
|
+@@ -153,8 +170,8 @@ public class TieredFileSegmentInputStream extends InputStream {
|
|||
|
|
+ int bufIndex = curReadBufferIndex;
|
|||
|
|
+ int posInCurBuffer = readPosInCurBuffer;
|
|||
|
|
+ ByteBuffer curBuf = curBuffer;
|
|||
|
|
+- while (needRead > 0 && bufIndex < uploadBufferList.size()) {
|
|||
|
|
+- curBuf = uploadBufferList.get(bufIndex);
|
|||
|
|
++ while (needRead > 0 && bufIndex < bufferList.size()) {
|
|||
|
|
++ curBuf = bufferList.get(bufIndex);
|
|||
|
|
+ int remaining = curBuf.remaining() - posInCurBuffer;
|
|||
|
|
+ int readLen = Math.min(remaining, needRead);
|
|||
|
|
+ // read from curBuf
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java
|
|||
|
|
+similarity index 54%
|
|||
|
|
+rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java
|
|||
|
|
+rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java
|
|||
|
|
+index d0c983fd4..a90baff3a 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java
|
|||
|
|
+@@ -15,30 +15,34 @@
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+
|
|||
|
|
+-package org.apache.rocketmq.tieredstore.provider.inputstream;
|
|||
|
|
++package org.apache.rocketmq.tieredstore.provider.stream;
|
|||
|
|
+
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.FileSegmentType;
|
|||
|
|
+
|
|||
|
|
+-public class TieredFileSegmentInputStreamFactory {
|
|||
|
|
++public class FileSegmentInputStreamFactory {
|
|||
|
|
+
|
|||
|
|
+- public static TieredFileSegmentInputStream build(FileSegmentType fileType,
|
|||
|
|
+- long startOffset, List<ByteBuffer> uploadBufferList, ByteBuffer codaBuffer, int contentLength) {
|
|||
|
|
++ public static FileSegmentInputStream build(
|
|||
|
|
++ FileSegmentType fileType, long offset, List<ByteBuffer> bufferList, ByteBuffer byteBuffer, int length) {
|
|||
|
|
++
|
|||
|
|
++ if (bufferList == null) {
|
|||
|
|
++ throw new IllegalArgumentException("bufferList is null");
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
+ switch (fileType) {
|
|||
|
|
+ case COMMIT_LOG:
|
|||
|
|
+- return new TieredCommitLogInputStream(
|
|||
|
|
+- fileType, startOffset, uploadBufferList, codaBuffer, contentLength);
|
|||
|
|
++ return new CommitLogInputStream(
|
|||
|
|
++ fileType, offset, bufferList, byteBuffer, length);
|
|||
|
|
+ case CONSUME_QUEUE:
|
|||
|
|
+- return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength);
|
|||
|
|
++ return new FileSegmentInputStream(fileType, bufferList, length);
|
|||
|
|
+ case INDEX:
|
|||
|
|
+- if (uploadBufferList.size() != 1) {
|
|||
|
|
+- throw new IllegalArgumentException("uploadBufferList size in INDEX type input stream must be 1");
|
|||
|
|
++ if (bufferList.size() != 1) {
|
|||
|
|
++ throw new IllegalArgumentException("buffer block size must be 1 when file type is IndexFile");
|
|||
|
|
+ }
|
|||
|
|
+- return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength);
|
|||
|
|
++ return new FileSegmentInputStream(fileType, bufferList, length);
|
|||
|
|
+ default:
|
|||
|
|
+- throw new IllegalArgumentException("fileType is not supported");
|
|||
|
|
++ throw new IllegalArgumentException("file type is not supported");
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java
|
|||
|
|
+index 8601392e7..2451199c2 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java
|
|||
|
|
+@@ -130,36 +130,36 @@ public class TieredMessageStoreTest {
|
|||
|
|
+ // TieredStorageLevel.DISABLE
|
|||
|
|
+ properties.setProperty("tieredStorageLevel", "0");
|
|||
|
|
+ configuration.update(properties);
|
|||
|
|
+- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
+
|
|||
|
|
+ // TieredStorageLevel.NOT_IN_DISK
|
|||
|
|
+ properties.setProperty("tieredStorageLevel", "1");
|
|||
|
|
+ configuration.update(properties);
|
|||
|
|
+ when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false);
|
|||
|
|
+- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
+
|
|||
|
|
+ when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
|
|||
|
|
+- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
+
|
|||
|
|
+ // TieredStorageLevel.NOT_IN_MEM
|
|||
|
|
+ properties.setProperty("tieredStorageLevel", "2");
|
|||
|
|
+ configuration.update(properties);
|
|||
|
|
+ Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false);
|
|||
|
|
+ Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true);
|
|||
|
|
+- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
+
|
|||
|
|
+ Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
|
|||
|
|
+ Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(false);
|
|||
|
|
+- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
+
|
|||
|
|
+ Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
|
|||
|
|
+ Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true);
|
|||
|
|
+- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
+
|
|||
|
|
+ // TieredStorageLevel.FORCE
|
|||
|
|
+ properties.setProperty("tieredStorageLevel", "3");
|
|||
|
|
+ configuration.update(properties);
|
|||
|
|
+- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java
|
|||
|
|
+index cc39cfbfc..7a4d05969 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java
|
|||
|
|
+@@ -24,6 +24,7 @@ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.FileSegmentType;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.metadata.FileSegmentMetadata;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
|
|||
|
|
+@@ -55,6 +56,7 @@ public class TieredFlatFileTest {
|
|||
|
|
+ public void tearDown() throws IOException {
|
|||
|
|
+ TieredStoreTestUtil.destroyMetadataStore();
|
|||
|
|
+ TieredStoreTestUtil.destroyTempDir(storePath);
|
|||
|
|
++ TieredStoreExecutor.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private List<FileSegmentMetadata> getSegmentMetadataList(TieredMetadataStore metadataStore) {
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java
|
|||
|
|
+index 262d6645b..2da72bc7a 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java
|
|||
|
|
+@@ -87,5 +87,7 @@ public class TieredIndexFileTest {
|
|||
|
|
+
|
|||
|
|
+ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join();
|
|||
|
|
+ Assert.assertEquals(1, indexList.size());
|
|||
|
|
++
|
|||
|
|
++ indexFile.destroy();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java
|
|||
|
|
+similarity index 82%
|
|||
|
|
+rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java
|
|||
|
|
+rename to tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java
|
|||
|
|
+index a6566b7de..3bbe41dd4 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java
|
|||
|
|
+@@ -20,13 +20,13 @@ package org.apache.rocketmq.tieredstore.provider;
|
|||
|
|
+ import java.io.InputStream;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
|
|||
|
|
+
|
|||
|
|
+-public class MockTieredFileSegmentInputStream extends TieredFileSegmentInputStream {
|
|||
|
|
++public class MockFileSegmentInputStream extends FileSegmentInputStream {
|
|||
|
|
+
|
|||
|
|
+ private final InputStream inputStream;
|
|||
|
|
+
|
|||
|
|
+- public MockTieredFileSegmentInputStream(InputStream inputStream) {
|
|||
|
|
++ public MockFileSegmentInputStream(InputStream inputStream) {
|
|||
|
|
+ super(null, null, Integer.MAX_VALUE);
|
|||
|
|
+ this.inputStream = inputStream;
|
|||
|
|
+ }
|
|||
|
|
+@@ -43,7 +43,7 @@ public class MockTieredFileSegmentInputStream extends TieredFileSegmentInputStre
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+- public List<ByteBuffer> getUploadBufferList() {
|
|||
|
|
++ public List<ByteBuffer> getBufferList() {
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java
|
|||
|
|
+index a2554ba3d..743d9182c 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java
|
|||
|
|
+@@ -28,8 +28,8 @@ import java.util.Random;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.FileSegmentType;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.file.TieredCommitLog;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue;
|
|||
|
|
+-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream;
|
|||
|
|
+-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+@@ -57,7 +57,7 @@ public class TieredFileSegmentInputStreamTest {
|
|||
|
|
+ bufferSize += byteBuffer.remaining();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- // build expected byte buffer for verifying the TieredFileSegmentInputStream
|
|||
|
|
++ // build expected byte buffer for verifying the FileSegmentInputStream
|
|||
|
|
+ ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize);
|
|||
|
|
+ for (ByteBuffer byteBuffer : uploadBufferList) {
|
|||
|
|
+ expectedByteBuffer.put(byteBuffer);
|
|||
|
|
+@@ -74,7 +74,7 @@ public class TieredFileSegmentInputStreamTest {
|
|||
|
|
+ int[] batchReadSizeTestSet = {
|
|||
|
|
+ MessageBufferUtil.PHYSICAL_OFFSET_POSITION - 1, MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtil.PHYSICAL_OFFSET_POSITION + 1, MSG_LEN - 1, MSG_LEN, MSG_LEN + 1
|
|||
|
|
+ };
|
|||
|
|
+- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build(
|
|||
|
|
++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build(
|
|||
|
|
+ FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), finalBufferSize, batchReadSizeTestSet);
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+@@ -98,7 +98,7 @@ public class TieredFileSegmentInputStreamTest {
|
|||
|
|
+ int codaBufferSize = codaBuffer.remaining();
|
|||
|
|
+ bufferSize += codaBufferSize;
|
|||
|
|
+
|
|||
|
|
+- // build expected byte buffer for verifying the TieredFileSegmentInputStream
|
|||
|
|
++ // build expected byte buffer for verifying the FileSegmentInputStream
|
|||
|
|
+ ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize);
|
|||
|
|
+ for (ByteBuffer byteBuffer : uploadBufferList) {
|
|||
|
|
+ expectedByteBuffer.put(byteBuffer);
|
|||
|
|
+@@ -119,7 +119,7 @@ public class TieredFileSegmentInputStreamTest {
|
|||
|
|
+ MSG_LEN - 1, MSG_LEN, MSG_LEN + 1,
|
|||
|
|
+ bufferSize - 1, bufferSize, bufferSize + 1
|
|||
|
|
+ };
|
|||
|
|
+- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build(
|
|||
|
|
++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build(
|
|||
|
|
+ FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, codaBuffer, finalBufferSize), finalBufferSize, batchReadSizeTestSet);
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+@@ -134,7 +134,7 @@ public class TieredFileSegmentInputStreamTest {
|
|||
|
|
+ bufferSize += byteBuffer.remaining();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- // build expected byte buffer for verifying the TieredFileSegmentInputStream
|
|||
|
|
++ // build expected byte buffer for verifying the FileSegmentInputStream
|
|||
|
|
+ ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize);
|
|||
|
|
+ for (ByteBuffer byteBuffer : uploadBufferList) {
|
|||
|
|
+ expectedByteBuffer.put(byteBuffer);
|
|||
|
|
+@@ -143,7 +143,7 @@ public class TieredFileSegmentInputStreamTest {
|
|||
|
|
+
|
|||
|
|
+ int finalBufferSize = bufferSize;
|
|||
|
|
+ int[] batchReadSizeTestSet = {TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE - 1, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE + 1};
|
|||
|
|
+- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build(
|
|||
|
|
++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build(
|
|||
|
|
+ FileSegmentType.CONSUME_QUEUE, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), bufferSize, batchReadSizeTestSet);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -156,16 +156,16 @@ public class TieredFileSegmentInputStreamTest {
|
|||
|
|
+ byteBuffer.flip();
|
|||
|
|
+ List<ByteBuffer> uploadBufferList = Arrays.asList(byteBuffer);
|
|||
|
|
+
|
|||
|
|
+- // build expected byte buffer for verifying the TieredFileSegmentInputStream
|
|||
|
|
++ // build expected byte buffer for verifying the FileSegmentInputStream
|
|||
|
|
+ ByteBuffer expectedByteBuffer = byteBuffer.slice();
|
|||
|
|
+
|
|||
|
|
+- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build(
|
|||
|
|
++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build(
|
|||
|
|
+ FileSegmentType.INDEX, COMMIT_LOG_START_OFFSET, uploadBufferList, null, byteBuffer.limit()), byteBuffer.limit(), new int[] {23, 24, 25});
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier<TieredFileSegmentInputStream> constructor,
|
|||
|
|
++ private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier<FileSegmentInputStream> constructor,
|
|||
|
|
+ int bufferSize, int[] readBatchSizeTestSet) {
|
|||
|
|
+- TieredFileSegmentInputStream inputStream = constructor.get();
|
|||
|
|
++ FileSegmentInputStream inputStream = constructor.get();
|
|||
|
|
+
|
|||
|
|
+ // verify
|
|||
|
|
+ verifyInputStream(inputStream, expectedByteBuffer);
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java
|
|||
|
|
+index 4cd83e0d2..a655710a5 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java
|
|||
|
|
+@@ -116,13 +116,22 @@ public class TieredFileSegmentTest {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+- public void testCommitFailed() {
|
|||
|
|
++ public void testCommitFailedThenSuccess() {
|
|||
|
|
+ long startTime = System.currentTimeMillis();
|
|||
|
|
+ MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG);
|
|||
|
|
+ long lastSize = segment.getSize();
|
|||
|
|
+- segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0);
|
|||
|
|
+- segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0);
|
|||
|
|
+-
|
|||
|
|
++ segment.setCheckSize(false);
|
|||
|
|
++ segment.initPosition(lastSize);
|
|||
|
|
++ segment.setSize((int) lastSize);
|
|||
|
|
++
|
|||
|
|
++ ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong(
|
|||
|
|
++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize);
|
|||
|
|
++ ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong(
|
|||
|
|
++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN);
|
|||
|
|
++ segment.append(buffer1, 0);
|
|||
|
|
++ segment.append(buffer2, 0);
|
|||
|
|
++
|
|||
|
|
++ // Mock new message arrive
|
|||
|
|
+ segment.blocker = new CompletableFuture<>();
|
|||
|
|
+ new Thread(() -> {
|
|||
|
|
+ try {
|
|||
|
|
+@@ -131,20 +140,88 @@ public class TieredFileSegmentTest {
|
|||
|
|
+ Assert.fail(e.getMessage());
|
|||
|
|
+ }
|
|||
|
|
+ ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
|
|||
|
|
++ buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2);
|
|||
|
|
+ buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime);
|
|||
|
|
+ segment.append(buffer, 0);
|
|||
|
|
+ segment.blocker.complete(false);
|
|||
|
|
+ }).start();
|
|||
|
|
+
|
|||
|
|
++ // Commit failed
|
|||
|
|
+ segment.commit();
|
|||
|
|
+ segment.blocker.join();
|
|||
|
|
++ segment.blocker = null;
|
|||
|
|
++
|
|||
|
|
++ // Copy data and assume commit success
|
|||
|
|
++ segment.getMemStore().put(buffer1);
|
|||
|
|
++ segment.getMemStore().put(buffer2);
|
|||
|
|
++ segment.setSize((int) (lastSize + MessageBufferUtilTest.MSG_LEN * 2));
|
|||
|
|
+
|
|||
|
|
+- segment.blocker = new CompletableFuture<>();
|
|||
|
|
+- segment.blocker.complete(true);
|
|||
|
|
+ segment.commit();
|
|||
|
|
++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition());
|
|||
|
|
++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset());
|
|||
|
|
++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset());
|
|||
|
|
++
|
|||
|
|
++ ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN);
|
|||
|
|
++ Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1));
|
|||
|
|
++
|
|||
|
|
++ ByteBuffer msg2 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN);
|
|||
|
|
++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtil.getCommitLogOffset(msg2));
|
|||
|
|
++
|
|||
|
|
++ ByteBuffer msg3 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtilTest.MSG_LEN);
|
|||
|
|
++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtil.getCommitLogOffset(msg3));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testCommitFailed3Times() {
|
|||
|
|
++ long startTime = System.currentTimeMillis();
|
|||
|
|
++ MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG);
|
|||
|
|
++ long lastSize = segment.getSize();
|
|||
|
|
++ segment.setCheckSize(false);
|
|||
|
|
++ segment.initPosition(lastSize);
|
|||
|
|
++ segment.setSize((int) lastSize);
|
|||
|
|
++
|
|||
|
|
++ ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong(
|
|||
|
|
++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize);
|
|||
|
|
++ ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong(
|
|||
|
|
++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN);
|
|||
|
|
++ segment.append(buffer1, 0);
|
|||
|
|
++ segment.append(buffer2, 0);
|
|||
|
|
++
|
|||
|
|
++ // Mock new message arrive
|
|||
|
|
++ segment.blocker = new CompletableFuture<>();
|
|||
|
|
++ new Thread(() -> {
|
|||
|
|
++ try {
|
|||
|
|
++ Thread.sleep(3000);
|
|||
|
|
++ } catch (InterruptedException e) {
|
|||
|
|
++ Assert.fail(e.getMessage());
|
|||
|
|
++ }
|
|||
|
|
++ ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
|
|||
|
|
++ buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2);
|
|||
|
|
++ buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime);
|
|||
|
|
++ segment.append(buffer, 0);
|
|||
|
|
++ segment.blocker.complete(false);
|
|||
|
|
++ }).start();
|
|||
|
|
++
|
|||
|
|
++ for (int i = 0; i < 3; i++) {
|
|||
|
|
++ segment.commit();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ Assert.assertEquals(lastSize, segment.getCommitPosition());
|
|||
|
|
++ Assert.assertEquals(baseOffset + lastSize, segment.getCommitOffset());
|
|||
|
|
++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset());
|
|||
|
|
++
|
|||
|
|
++ segment.blocker.join();
|
|||
|
|
++ segment.blocker = null;
|
|||
|
|
+
|
|||
|
|
++ segment.commit();
|
|||
|
|
++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitPosition());
|
|||
|
|
++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitOffset());
|
|||
|
|
+ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset());
|
|||
|
|
++
|
|||
|
|
++ segment.commit();
|
|||
|
|
++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition());
|
|||
|
|
+ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset());
|
|||
|
|
++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset());
|
|||
|
|
+
|
|||
|
|
+ ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN);
|
|||
|
|
+ Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1));
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java
|
|||
|
|
+index cb155cf8f..80ad41f68 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java
|
|||
|
|
+@@ -23,7 +23,7 @@ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.FileSegmentType;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
|
|||
|
|
+-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+
|
|||
|
|
+@@ -33,6 +33,8 @@ public class MemoryFileSegment extends TieredFileSegment {
|
|||
|
|
+
|
|||
|
|
+ public CompletableFuture<Boolean> blocker;
|
|||
|
|
+
|
|||
|
|
++ protected int size = 0;
|
|||
|
|
++
|
|||
|
|
+ protected boolean checkSize = true;
|
|||
|
|
+
|
|||
|
|
+ public MemoryFileSegment(FileSegmentType fileType, MessageQueue messageQueue, long baseOffset,
|
|||
|
|
+@@ -56,6 +58,18 @@ public class MemoryFileSegment extends TieredFileSegment {
|
|||
|
|
+ memStore.position((int) getSize());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public boolean isCheckSize() {
|
|||
|
|
++ return checkSize;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setCheckSize(boolean checkSize) {
|
|||
|
|
++ this.checkSize = checkSize;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public ByteBuffer getMemStore() {
|
|||
|
|
++ return memStore;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public String getPath() {
|
|||
|
|
+ return filePath;
|
|||
|
|
+@@ -66,7 +80,11 @@ public class MemoryFileSegment extends TieredFileSegment {
|
|||
|
|
+ if (checkSize) {
|
|||
|
|
+ return 1000;
|
|||
|
|
+ }
|
|||
|
|
+- return 0;
|
|||
|
|
++ return size;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setSize(int size) {
|
|||
|
|
++ this.size = size;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+@@ -85,11 +103,11 @@ public class MemoryFileSegment extends TieredFileSegment {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<Boolean> commit0(
|
|||
|
|
+- TieredFileSegmentInputStream inputStream, long position, int length, boolean append) {
|
|||
|
|
++ FileSegmentInputStream inputStream, long position, int length, boolean append) {
|
|||
|
|
+
|
|||
|
|
+ try {
|
|||
|
|
+ if (blocker != null && !blocker.get()) {
|
|||
|
|
+- throw new IllegalStateException();
|
|||
|
|
++ throw new IllegalStateException("Commit Exception for Memory Test");
|
|||
|
|
+ }
|
|||
|
|
+ } catch (InterruptedException | ExecutionException e) {
|
|||
|
|
+ Assert.fail(e.getMessage());
|
|||
|
|
+@@ -98,7 +116,6 @@ public class MemoryFileSegment extends TieredFileSegment {
|
|||
|
|
+ Assert.assertTrue(!checkSize || position >= getSize());
|
|||
|
|
+
|
|||
|
|
+ byte[] buffer = new byte[1024];
|
|||
|
|
+-
|
|||
|
|
+ int startPos = memStore.position();
|
|||
|
|
+ try {
|
|||
|
|
+ int len;
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java
|
|||
|
|
+index 8ac330b37..630fd2223 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java
|
|||
|
|
+@@ -22,7 +22,7 @@ import java.util.concurrent.ExecutionException;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.FileSegmentType;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
|
|||
|
|
+-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+
|
|||
|
|
+@@ -46,7 +46,7 @@ public class MemoryFileSegmentWithoutCheck extends MemoryFileSegment {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+- public CompletableFuture<Boolean> commit0(TieredFileSegmentInputStream inputStream, long position, int length,
|
|||
|
|
++ public CompletableFuture<Boolean> commit0(FileSegmentInputStream inputStream, long position, int length,
|
|||
|
|
+ boolean append) {
|
|||
|
|
+ try {
|
|||
|
|
+ if (blocker != null && !blocker.get()) {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From d000ef947d7c99918ceba0fa451c1e29fd84ba07 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yuz10 <845238369@qq.com>
|
|||
|
|
+Date: Thu, 31 Aug 2023 09:41:33 +0800
|
|||
|
|
+Subject: [PATCH 3/7] [ISSUE #7283] Incorrect dledger commitlog min offset
|
|||
|
|
+ after mappedFile re delete failed (#7284)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/store/dledger/DLedgerCommitLog.java | 7 ++++++-
|
|||
|
|
+ 1 file changed, 6 insertions(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
|
|||
|
|
+index ec5e86d70..d5f6acdc0 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
|
|||
|
|
+@@ -162,7 +162,12 @@ public class DLedgerCommitLog extends CommitLog {
|
|||
|
|
+ if (!mappedFileQueue.getMappedFiles().isEmpty()) {
|
|||
|
|
+ return mappedFileQueue.getMinOffset();
|
|||
|
|
+ }
|
|||
|
|
+- return dLedgerFileList.getMinOffset();
|
|||
|
|
++ for (MmapFile file : dLedgerFileList.getMappedFiles()) {
|
|||
|
|
++ if (file.isAvailable()) {
|
|||
|
|
++ return file.getFileFromOffset() + file.getStartPosition();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From f82718ae3b77a16b553c03f672dc971a2d5d48fa Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: cnScarb <jjhfen00@163.com>
|
|||
|
|
+Date: Thu, 31 Aug 2023 15:50:10 +0800
|
|||
|
|
+Subject: [PATCH 4/7] [ISSUE #7208] fix: when deleting topic also delete its
|
|||
|
|
+ pop retry topic (#7209)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../processor/AdminBrokerProcessor.java | 24 ++++++++++---
|
|||
|
|
+ .../processor/AdminBrokerProcessorTest.java | 36 +++++++++++++++++++
|
|||
|
|
+ 2 files changed, 55 insertions(+), 5 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
+index bbddcec2d..8fbcd3c94 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
+@@ -51,6 +51,7 @@ import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
|
|||
|
|
+ import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;
|
|||
|
|
+ import org.apache.rocketmq.common.AclConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.BrokerConfig;
|
|||
|
|
++import org.apache.rocketmq.common.KeyBuilder;
|
|||
|
|
+ import org.apache.rocketmq.common.LockCallback;
|
|||
|
|
+ import org.apache.rocketmq.common.MQVersion;
|
|||
|
|
+ import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+@@ -542,16 +543,29 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- this.brokerController.getTopicConfigManager().deleteTopicConfig(requestHeader.getTopic());
|
|||
|
|
+- this.brokerController.getTopicQueueMappingManager().delete(requestHeader.getTopic());
|
|||
|
|
+- this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(requestHeader.getTopic());
|
|||
|
|
+- this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(requestHeader.getTopic());
|
|||
|
|
+- this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(requestHeader.getTopic()));
|
|||
|
|
++ final Set<String> groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic);
|
|||
|
|
++ // delete pop retry topics first
|
|||
|
|
++ for (String group : groups) {
|
|||
|
|
++ final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group);
|
|||
|
|
++ if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) {
|
|||
|
|
++ deleteTopicInBroker(popRetryTopic);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ // delete topic
|
|||
|
|
++ deleteTopicInBroker(topic);
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+ response.setRemark(null);
|
|||
|
|
+ return response;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ private void deleteTopicInBroker(String topic) {
|
|||
|
|
++ this.brokerController.getTopicConfigManager().deleteTopicConfig(topic);
|
|||
|
|
++ this.brokerController.getTopicQueueMappingManager().delete(topic);
|
|||
|
|
++ this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(topic);
|
|||
|
|
++ this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(topic);
|
|||
|
|
++ this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(topic));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx,
|
|||
|
|
+ RemotingCommand request) throws RemotingCommandException {
|
|||
|
|
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
|
|||
|
|
+index d33a217f7..9d17011b6 100644
|
|||
|
|
+--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
|
|||
|
|
++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
|
|||
|
|
+@@ -29,6 +29,7 @@ import java.util.HashMap;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+ import java.util.Properties;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
++import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.atomic.LongAdder;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+@@ -41,6 +42,7 @@ import org.apache.rocketmq.broker.topic.RocksDBTopicConfigManager;
|
|||
|
|
+ import org.apache.rocketmq.broker.topic.TopicConfigManager;
|
|||
|
|
+ import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+ import org.apache.rocketmq.common.BrokerConfig;
|
|||
|
|
++import org.apache.rocketmq.common.KeyBuilder;
|
|||
|
|
+ import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+ import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.TopicFilterType;
|
|||
|
|
+@@ -90,8 +92,11 @@ import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.any;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyInt;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyLong;
|
|||
|
|
++import static org.mockito.ArgumentMatchers.anySet;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyString;
|
|||
|
|
+ import static org.mockito.Mockito.mock;
|
|||
|
|
++import static org.mockito.Mockito.times;
|
|||
|
|
++import static org.mockito.Mockito.verify;
|
|||
|
|
+ import static org.mockito.Mockito.when;
|
|||
|
|
+
|
|||
|
|
+ @RunWith(MockitoJUnitRunner.class)
|
|||
|
|
+@@ -321,6 +326,37 @@ public class AdminBrokerProcessorTest {
|
|||
|
|
+ "please execute it from master broker.");
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testDeleteWithPopRetryTopic() throws Exception {
|
|||
|
|
++ String topic = "topicA";
|
|||
|
|
++ String anotherTopic = "another_topicA";
|
|||
|
|
++
|
|||
|
|
++ topicConfigManager = mock(TopicConfigManager.class);
|
|||
|
|
++ when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
|
|||
|
|
++ final ConcurrentHashMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();
|
|||
|
|
++ topicConfigTable.put(topic, new TopicConfig());
|
|||
|
|
++ topicConfigTable.put(KeyBuilder.buildPopRetryTopic(topic, "cid1"), new TopicConfig());
|
|||
|
|
++
|
|||
|
|
++ topicConfigTable.put(anotherTopic, new TopicConfig());
|
|||
|
|
++ topicConfigTable.put(KeyBuilder.buildPopRetryTopic(anotherTopic, "cid2"), new TopicConfig());
|
|||
|
|
++ when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigTable);
|
|||
|
|
++ when(topicConfigManager.selectTopicConfig(anyString())).thenAnswer(invocation -> {
|
|||
|
|
++ final String selectTopic = invocation.getArgument(0);
|
|||
|
|
++ return topicConfigManager.getTopicConfigTable().get(selectTopic);
|
|||
|
|
++ });
|
|||
|
|
++
|
|||
|
|
++ when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
|
|||
|
|
++ when(consumerOffsetManager.whichGroupByTopic(topic)).thenReturn(Sets.newHashSet("cid1"));
|
|||
|
|
++
|
|||
|
|
++ RemotingCommand request = buildDeleteTopicRequest(topic);
|
|||
|
|
++ RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
|
|||
|
|
++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
|
|||
|
|
++
|
|||
|
|
++ verify(topicConfigManager).deleteTopicConfig(topic);
|
|||
|
|
++ verify(topicConfigManager).deleteTopicConfig(KeyBuilder.buildPopRetryTopic(topic, "cid1"));
|
|||
|
|
++ verify(messageStore, times(2)).deleteTopics(anySet());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetAllTopicConfigInRocksdb() throws Exception {
|
|||
|
|
+ if (notToBeExecuted()) {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 31d10385d1616445478104ce9ef463a8c4852ba2 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: guyinyou <36399867+guyinyou@users.noreply.github.com>
|
|||
|
|
+Date: Mon, 4 Sep 2023 14:09:32 +0800
|
|||
|
|
+Subject: [PATCH 5/7] [ISSUE #7289] Fixed asynchronous send backpressure
|
|||
|
|
+ capability
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: guyinyou <guyinyou.gyy@alibaba-inc.com>
|
|||
|
|
+---
|
|||
|
|
+ .../impl/producer/DefaultMQProducerImpl.java | 77 +++++++++++++------
|
|||
|
|
+ 1 file changed, 53 insertions(+), 24 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
|
|||
|
|
+index bbbb17b07..2d6b83ac2 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
|
|||
|
|
+@@ -547,6 +547,8 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ @Deprecated
|
|||
|
|
+ public void send(final Message msg, final SendCallback sendCallback, final long timeout)
|
|||
|
|
+ throws MQClientException, RemotingException, InterruptedException {
|
|||
|
|
++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback);
|
|||
|
|
++
|
|||
|
|
+ final long beginStartTime = System.currentTimeMillis();
|
|||
|
|
+ Runnable runnable = new Runnable() {
|
|||
|
|
+ @Override
|
|||
|
|
+@@ -554,20 +556,53 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ long costTime = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
+ if (timeout > costTime) {
|
|||
|
|
+ try {
|
|||
|
|
+- sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime);
|
|||
|
|
++ sendDefaultImpl(msg, CommunicationMode.ASYNC, newCallBack, timeout - costTime);
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- sendCallback.onException(e);
|
|||
|
|
++ newCallBack.onException(e);
|
|||
|
|
+ }
|
|||
|
|
+ } else {
|
|||
|
|
+- sendCallback.onException(
|
|||
|
|
++ newCallBack.onException(
|
|||
|
|
+ new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout"));
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ };
|
|||
|
|
+- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime);
|
|||
|
|
++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void executeAsyncMessageSend(Runnable runnable, final Message msg, final SendCallback sendCallback,
|
|||
|
|
++ class BackpressureSendCallBack implements SendCallback {
|
|||
|
|
++ public boolean isSemaphoreAsyncSizeAquired = false;
|
|||
|
|
++ public boolean isSemaphoreAsyncNumAquired = false;
|
|||
|
|
++ public int msgLen;
|
|||
|
|
++ private final SendCallback sendCallback;
|
|||
|
|
++
|
|||
|
|
++ public BackpressureSendCallBack(final SendCallback sendCallback) {
|
|||
|
|
++ this.sendCallback = sendCallback;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onSuccess(SendResult sendResult) {
|
|||
|
|
++ if (isSemaphoreAsyncSizeAquired) {
|
|||
|
|
++ semaphoreAsyncSendSize.release(msgLen);
|
|||
|
|
++ }
|
|||
|
|
++ if (isSemaphoreAsyncNumAquired) {
|
|||
|
|
++ semaphoreAsyncSendNum.release();
|
|||
|
|
++ }
|
|||
|
|
++ sendCallback.onSuccess(sendResult);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onException(Throwable e) {
|
|||
|
|
++ if (isSemaphoreAsyncSizeAquired) {
|
|||
|
|
++ semaphoreAsyncSendSize.release(msgLen);
|
|||
|
|
++ }
|
|||
|
|
++ if (isSemaphoreAsyncNumAquired) {
|
|||
|
|
++ semaphoreAsyncSendNum.release();
|
|||
|
|
++ }
|
|||
|
|
++ sendCallback.onException(e);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final BackpressureSendCallBack sendCallback,
|
|||
|
|
+ final long timeout, final long beginStartTime)
|
|||
|
|
+ throws MQClientException, InterruptedException {
|
|||
|
|
+ ExecutorService executor = this.getAsyncSenderExecutor();
|
|||
|
|
+@@ -595,7 +630,9 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
++ sendCallback.isSemaphoreAsyncSizeAquired = isSemaphoreAsyncSizeAquired;
|
|||
|
|
++ sendCallback.isSemaphoreAsyncNumAquired = isSemaphoreAsyncNumAquired;
|
|||
|
|
++ sendCallback.msgLen = msgLen;
|
|||
|
|
+ executor.submit(runnable);
|
|||
|
|
+ } catch (RejectedExecutionException e) {
|
|||
|
|
+ if (isEnableBackpressureForAsyncMode) {
|
|||
|
|
+@@ -603,15 +640,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ } else {
|
|||
|
|
+ throw new MQClientException("executor rejected ", e);
|
|||
|
|
+ }
|
|||
|
|
+- } finally {
|
|||
|
|
+- if (isSemaphoreAsyncSizeAquired) {
|
|||
|
|
+- semaphoreAsyncSendSize.release(msgLen);
|
|||
|
|
+- }
|
|||
|
|
+- if (isSemaphoreAsyncNumAquired) {
|
|||
|
|
+- semaphoreAsyncSendNum.release();
|
|||
|
|
+- }
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg,
|
|||
|
|
+@@ -1188,7 +1217,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ @Deprecated
|
|||
|
|
+ public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout)
|
|||
|
|
+ throws MQClientException, RemotingException, InterruptedException {
|
|||
|
|
+-
|
|||
|
|
++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback);
|
|||
|
|
+ final long beginStartTime = System.currentTimeMillis();
|
|||
|
|
+ Runnable runnable = new Runnable() {
|
|||
|
|
+ @Override
|
|||
|
|
+@@ -1203,22 +1232,22 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ long costTime = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
+ if (timeout > costTime) {
|
|||
|
|
+ try {
|
|||
|
|
+- sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null,
|
|||
|
|
++ sendKernelImpl(msg, mq, CommunicationMode.ASYNC, newCallBack, null,
|
|||
|
|
+ timeout - costTime);
|
|||
|
|
+ } catch (MQBrokerException e) {
|
|||
|
|
+ throw new MQClientException("unknown exception", e);
|
|||
|
|
+ }
|
|||
|
|
+ } else {
|
|||
|
|
+- sendCallback.onException(new RemotingTooMuchRequestException("call timeout"));
|
|||
|
|
++ newCallBack.onException(new RemotingTooMuchRequestException("call timeout"));
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- sendCallback.onException(e);
|
|||
|
|
++ newCallBack.onException(e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ };
|
|||
|
|
+
|
|||
|
|
+- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime);
|
|||
|
|
++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+@@ -1315,7 +1344,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ public void send(final Message msg, final MessageQueueSelector selector, final Object arg,
|
|||
|
|
+ final SendCallback sendCallback, final long timeout)
|
|||
|
|
+ throws MQClientException, RemotingException, InterruptedException {
|
|||
|
|
+-
|
|||
|
|
++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback);
|
|||
|
|
+ final long beginStartTime = System.currentTimeMillis();
|
|||
|
|
+ Runnable runnable = new Runnable() {
|
|||
|
|
+ @Override
|
|||
|
|
+@@ -1324,21 +1353,21 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ if (timeout > costTime) {
|
|||
|
|
+ try {
|
|||
|
|
+ try {
|
|||
|
|
+- sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback,
|
|||
|
|
++ sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, newCallBack,
|
|||
|
|
+ timeout - costTime);
|
|||
|
|
+ } catch (MQBrokerException e) {
|
|||
|
|
+ throw new MQClientException("unknown exception", e);
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- sendCallback.onException(e);
|
|||
|
|
++ newCallBack.onException(e);
|
|||
|
|
+ }
|
|||
|
|
+ } else {
|
|||
|
|
+- sendCallback.onException(new RemotingTooMuchRequestException("call timeout"));
|
|||
|
|
++ newCallBack.onException(new RemotingTooMuchRequestException("call timeout"));
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ };
|
|||
|
|
+- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime);
|
|||
|
|
++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From d67b9d64cbd53824798af57ba18770e0fcefa37a Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yuz10 <845238369@qq.com>
|
|||
|
|
+Date: Wed, 6 Sep 2023 14:07:23 +0800
|
|||
|
|
+Subject: [PATCH 6/7] [ISSUE #7302] Fix singleTopicRegister code deleted in
|
|||
|
|
+ merge
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/broker/topic/TopicConfigManager.java | 6 +++++-
|
|||
|
|
+ 1 file changed, 5 insertions(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
+index 1c3b9711f..4e3c1736c 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
+@@ -330,7 +330,11 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ log.error("createTopicIfAbsent ", e);
|
|||
|
|
+ }
|
|||
|
|
+ if (createNew && register) {
|
|||
|
|
+- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
++ this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
++ } else {
|
|||
|
|
++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ return getTopicConfig(topicConfig.getTopicName());
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 37017dbaec5c521fd529ef4aecf3658092884f84 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lizhimins <707364882@qq.com>
|
|||
|
|
+Date: Wed, 6 Sep 2023 15:23:15 +0800
|
|||
|
|
+Subject: [PATCH 7/7] [ISSUE #7305] Fix metrics and transactional module not
|
|||
|
|
+ shutdown while broker offline cause coredump(#7307)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../java/org/apache/rocketmq/broker/BrokerController.java | 8 ++++++++
|
|||
|
|
+ .../queue/TransactionalMessageServiceImpl.java | 4 +++-
|
|||
|
|
+ 2 files changed, 11 insertions(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+index e8f943702..6aba70cb2 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+@@ -1302,6 +1302,10 @@ public class BrokerController {
|
|||
|
|
+ this.fastRemotingServer.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ if (this.brokerMetricsManager != null) {
|
|||
|
|
++ this.brokerMetricsManager.shutdown();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ if (this.brokerStatsManager != null) {
|
|||
|
|
+ this.brokerStatsManager.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+@@ -1324,6 +1328,10 @@ public class BrokerController {
|
|||
|
|
+ this.ackMessageProcessor.shutdownPopReviveService();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ if (this.transactionalMessageService != null) {
|
|||
|
|
++ this.transactionalMessageService.close();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ if (this.notificationProcessor != null) {
|
|||
|
|
+ this.notificationProcessor.getPopLongPollingService().shutdown();
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java
|
|||
|
|
+index 93fa725a9..48db828e0 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java
|
|||
|
|
+@@ -629,7 +629,9 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void close() {
|
|||
|
|
+-
|
|||
|
|
++ if (this.transactionalOpBatchService != null) {
|
|||
|
|
++ this.transactionalOpBatchService.shutdown();
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public Message getOpMessage(int queueId, String moreData) {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch016-backport-Optimize-fault-tolerant-mechanism.patch b/patch016-backport-Optimize-fault-tolerant-mechanism.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..9d8ef20a1
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch016-backport-Optimize-fault-tolerant-mechanism.patch
|
|||
|
|
@@ -0,0 +1,520 @@
|
|||
|
|
+From e11e29419f6e2d1d9673d0329e57b824ebf3da47 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lizhimins <707364882@qq.com>
|
|||
|
|
+Date: Wed, 6 Sep 2023 20:42:24 +0800
|
|||
|
|
+Subject: [PATCH 1/3] [ISSUE #7308] Adding topic blacklist and filter in tiered
|
|||
|
|
+ storage module (#7310)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../tieredstore/TieredDispatcher.java | 21 +++++++--
|
|||
|
|
+ .../tieredstore/TieredMessageStore.java | 1 +
|
|||
|
|
+ .../file/TieredFlatFileManager.java | 17 ++++---
|
|||
|
|
+ .../TieredStoreTopicBlackListFilter.java | 45 +++++++++++++++++++
|
|||
|
|
+ .../provider/TieredStoreTopicFilter.java | 25 +++++++++++
|
|||
|
|
+ .../TieredStoreTopicBlackListFilterTest.java | 36 +++++++++++++++
|
|||
|
|
+ 6 files changed, 136 insertions(+), 9 deletions(-)
|
|||
|
|
+ create mode 100644 tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java
|
|||
|
|
+ create mode 100644 tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java
|
|||
|
|
+ create mode 100644 tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
|
|||
|
|
+index 430c2b62e..766c559e9 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
|
|||
|
|
+@@ -48,6 +48,8 @@ import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicBlackListFilter;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicFilter;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.CQItemBufferUtil;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+@@ -56,6 +58,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch
|
|||
|
|
+
|
|||
|
|
+ private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
++ private TieredStoreTopicFilter topicFilter;
|
|||
|
|
+ private final String brokerName;
|
|||
|
|
+ private final MessageStore defaultStore;
|
|||
|
|
+ private final TieredMessageStoreConfig storeConfig;
|
|||
|
|
+@@ -70,15 +73,15 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch
|
|||
|
|
+ this.defaultStore = defaultStore;
|
|||
|
|
+ this.storeConfig = storeConfig;
|
|||
|
|
+ this.brokerName = storeConfig.getBrokerName();
|
|||
|
|
++ this.topicFilter = new TieredStoreTopicBlackListFilter();
|
|||
|
|
+ this.tieredFlatFileManager = TieredFlatFileManager.getInstance(storeConfig);
|
|||
|
|
+ this.dispatchRequestReadMap = new ConcurrentHashMap<>();
|
|||
|
|
+ this.dispatchRequestWriteMap = new ConcurrentHashMap<>();
|
|||
|
|
+ this.dispatchTaskLock = new ReentrantLock();
|
|||
|
|
+ this.dispatchWriteLock = new ReentrantLock();
|
|||
|
|
+- this.initScheduleTask();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private void initScheduleTask() {
|
|||
|
|
++ protected void initScheduleTask() {
|
|||
|
|
+ TieredStoreExecutor.commonScheduledExecutor.scheduleWithFixedDelay(() ->
|
|||
|
|
+ tieredFlatFileManager.deepCopyFlatFileToList().forEach(flatFile -> {
|
|||
|
|
+ if (!flatFile.getCompositeFlatFileLock().isLocked()) {
|
|||
|
|
+@@ -87,6 +90,14 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch
|
|||
|
|
+ }), 30, 10, TimeUnit.SECONDS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public TieredStoreTopicFilter getTopicFilter() {
|
|||
|
|
++ return topicFilter;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setTopicFilter(TieredStoreTopicFilter topicFilter) {
|
|||
|
|
++ this.topicFilter = topicFilter;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public void dispatch(DispatchRequest request) {
|
|||
|
|
+ if (stopped) {
|
|||
|
|
+@@ -94,7 +105,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ String topic = request.getTopic();
|
|||
|
|
+- if (TieredStoreUtil.isSystemTopic(topic)) {
|
|||
|
|
++ if (topicFilter != null && topicFilter.filterTopic(topic)) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -219,6 +230,10 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ if (topicFilter != null && topicFilter.filterTopic(flatFile.getMessageQueue().getTopic())) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ if (flatFile.getDispatchOffset() == -1L) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+index 78e855f36..9fb1b2f01 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+@@ -90,6 +90,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ boolean loadNextStore = next.load();
|
|||
|
|
+ boolean result = loadFlatFile && loadNextStore;
|
|||
|
|
+ if (result) {
|
|||
|
|
++ dispatcher.initScheduleTask();
|
|||
|
|
+ dispatcher.start();
|
|||
|
|
+ }
|
|||
|
|
+ return result;
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
|
|||
|
|
+index e9ae4a5a5..7c744af3b 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
|
|||
|
|
+@@ -134,21 +134,21 @@ public class TieredFlatFileManager {
|
|||
|
|
+ public void doCleanExpiredFile() {
|
|||
|
|
+ long expiredTimeStamp = System.currentTimeMillis() -
|
|||
|
|
+ TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime());
|
|||
|
|
+- Random random = new Random();
|
|||
|
|
+ for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) {
|
|||
|
|
+- int delay = random.nextInt(storeConfig.getMaxCommitJitter());
|
|||
|
|
+- TieredStoreExecutor.cleanExpiredFileExecutor.schedule(() -> {
|
|||
|
|
++ TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> {
|
|||
|
|
+ flatFile.getCompositeFlatFileLock().lock();
|
|||
|
|
+ try {
|
|||
|
|
+ flatFile.cleanExpiredFile(expiredTimeStamp);
|
|||
|
|
+ flatFile.destroyExpiredFile();
|
|||
|
|
+ if (flatFile.getConsumeQueueBaseOffset() == -1) {
|
|||
|
|
++ logger.info("Clean flatFile because file not initialized, topic={}, queueId={}",
|
|||
|
|
++ flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId());
|
|||
|
|
+ destroyCompositeFile(flatFile.getMessageQueue());
|
|||
|
|
+ }
|
|||
|
|
+ } finally {
|
|||
|
|
+ flatFile.getCompositeFlatFileLock().unlock();
|
|||
|
|
+ }
|
|||
|
|
+- }, delay, TimeUnit.MILLISECONDS);
|
|||
|
|
++ });
|
|||
|
|
+ }
|
|||
|
|
+ if (indexFile != null) {
|
|||
|
|
+ indexFile.cleanExpiredFile(expiredTimeStamp);
|
|||
|
|
+@@ -218,8 +218,13 @@ public class TieredFlatFileManager {
|
|||
|
|
+ storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId()));
|
|||
|
|
+ queueCount.incrementAndGet();
|
|||
|
|
+ });
|
|||
|
|
+- logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms",
|
|||
|
|
+- topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS));
|
|||
|
|
++
|
|||
|
|
++ if (queueCount.get() == 0L) {
|
|||
|
|
++ metadataStore.deleteTopic(topicMetadata.getTopic());
|
|||
|
|
++ } else {
|
|||
|
|
++ logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms",
|
|||
|
|
++ topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS));
|
|||
|
|
++ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e);
|
|||
|
|
+ } finally {
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..50adbb713
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java
|
|||
|
|
+@@ -0,0 +1,45 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++
|
|||
|
|
++package org.apache.rocketmq.tieredstore.provider;
|
|||
|
|
++
|
|||
|
|
++import java.util.HashSet;
|
|||
|
|
++import java.util.Set;
|
|||
|
|
++import org.apache.commons.lang3.StringUtils;
|
|||
|
|
++import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
++
|
|||
|
|
++public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter {
|
|||
|
|
++
|
|||
|
|
++ private final Set<String> topicBlackSet;
|
|||
|
|
++
|
|||
|
|
++ public TieredStoreTopicBlackListFilter() {
|
|||
|
|
++ this.topicBlackSet = new HashSet<>();
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean filterTopic(String topicName) {
|
|||
|
|
++ if (StringUtils.isBlank(topicName)) {
|
|||
|
|
++ return true;
|
|||
|
|
++ }
|
|||
|
|
++ return TieredStoreUtil.isSystemTopic(topicName) || topicBlackSet.contains(topicName);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void addTopicToWhiteList(String topicName) {
|
|||
|
|
++ this.topicBlackSet.add(topicName);
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..3f26b8b02
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java
|
|||
|
|
+@@ -0,0 +1,25 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++
|
|||
|
|
++package org.apache.rocketmq.tieredstore.provider;
|
|||
|
|
++
|
|||
|
|
++public interface TieredStoreTopicFilter {
|
|||
|
|
++
|
|||
|
|
++ boolean filterTopic(String topicName);
|
|||
|
|
++
|
|||
|
|
++ void addTopicToWhiteList(String topicName);
|
|||
|
|
++}
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..2bf48173c
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java
|
|||
|
|
+@@ -0,0 +1,36 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++package org.apache.rocketmq.tieredstore.provider;
|
|||
|
|
++
|
|||
|
|
++import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
++import org.junit.Assert;
|
|||
|
|
++import org.junit.Test;
|
|||
|
|
++
|
|||
|
|
++public class TieredStoreTopicBlackListFilterTest {
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void filterTopicTest() {
|
|||
|
|
++ TieredStoreTopicFilter topicFilter = new TieredStoreTopicBlackListFilter();
|
|||
|
|
++ Assert.assertTrue(topicFilter.filterTopic(""));
|
|||
|
|
++ Assert.assertTrue(topicFilter.filterTopic(TopicValidator.SYSTEM_TOPIC_PREFIX + "_Topic"));
|
|||
|
|
++
|
|||
|
|
++ String topicName = "WhiteTopic";
|
|||
|
|
++ Assert.assertFalse(topicFilter.filterTopic(topicName));
|
|||
|
|
++ topicFilter.addTopicToWhiteList(topicName);
|
|||
|
|
++ Assert.assertTrue(topicFilter.filterTopic(topicName));
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+\ No newline at end of file
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 628020537fa7035226bc8dcde9fa33d9d5df30ff Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: rongtong <jinrongtong5@163.com>
|
|||
|
|
+Date: Thu, 7 Sep 2023 16:17:47 +0800
|
|||
|
|
+Subject: [PATCH 2/3] [ISSUE #7293] Fix NPE when alter sync state set
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/controller/impl/manager/ReplicasInfoManager.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java
|
|||
|
|
+index b0a67531d..d83a690f9 100644
|
|||
|
|
+--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java
|
|||
|
|
++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java
|
|||
|
|
+@@ -104,7 +104,7 @@ public class ReplicasInfoManager {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ // Check master
|
|||
|
|
+- if (!syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) {
|
|||
|
|
++ if (syncStateInfo.getMasterBrokerId() == null || !syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) {
|
|||
|
|
+ String err = String.format("Rejecting alter syncStateSet request because the current leader is:{%s}, not {%s}",
|
|||
|
|
+ syncStateInfo.getMasterBrokerId(), request.getMasterBrokerId());
|
|||
|
|
+ LOGGER.error("{}", err);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 6fd0073d6475c539e8f4c30dc4f104a56a21d724 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ji Juntao <juntao.jjt@alibaba-inc.com>
|
|||
|
|
+Date: Thu, 7 Sep 2023 20:21:16 +0800
|
|||
|
|
+Subject: [PATCH 3/3] [ISSUE #7319] Optimize fault-tolerant mechanism for
|
|||
|
|
+ sending messages and hot update switch (#7320)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../impl/producer/DefaultMQProducerImpl.java | 8 ++------
|
|||
|
|
+ .../client/latency/LatencyFaultTolerance.java | 14 +++++++++++++
|
|||
|
|
+ .../latency/LatencyFaultToleranceImpl.java | 13 +++++++++++-
|
|||
|
|
+ .../client/latency/MQFaultStrategy.java | 20 +++++++------------
|
|||
|
|
+ .../proxy/service/route/MessageQueueView.java | 9 ---------
|
|||
|
|
+ .../service/route/TopicRouteService.java | 10 +++++++++-
|
|||
|
|
+ 6 files changed, 44 insertions(+), 30 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
|
|||
|
|
+index 2d6b83ac2..b0c212e46 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
|
|||
|
|
+@@ -263,9 +263,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ mQClientFactory.start();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- if (this.mqFaultStrategy.isStartDetectorEnable()) {
|
|||
|
|
+- this.mqFaultStrategy.startDetector();
|
|||
|
|
+- }
|
|||
|
|
++ this.mqFaultStrategy.startDetector();
|
|||
|
|
+
|
|||
|
|
+ log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(),
|
|||
|
|
+ this.defaultMQProducer.isSendMessageWithVIPChannel());
|
|||
|
|
+@@ -311,9 +309,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
|
|||
|
|
+ if (shutdownFactory) {
|
|||
|
|
+ this.mQClientFactory.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+- if (this.mqFaultStrategy.isStartDetectorEnable()) {
|
|||
|
|
+- this.mqFaultStrategy.shutdown();
|
|||
|
|
+- }
|
|||
|
|
++ this.mqFaultStrategy.shutdown();
|
|||
|
|
+ RequestFutureHolder.getInstance().shutdown(this);
|
|||
|
|
+ log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup());
|
|||
|
|
+ this.serviceState = ServiceState.SHUTDOWN_ALREADY;
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java
|
|||
|
|
+index 72d2f3450..17aaa266a 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java
|
|||
|
|
+@@ -89,4 +89,18 @@ public interface LatencyFaultTolerance<T> {
|
|||
|
|
+ * @param detectInterval each broker's detecting interval
|
|||
|
|
+ */
|
|||
|
|
+ void setDetectInterval(final int detectInterval);
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * Use it to set the detector work or not.
|
|||
|
|
++ *
|
|||
|
|
++ * @param startDetectorEnable set the detector's work status
|
|||
|
|
++ */
|
|||
|
|
++ void setStartDetectorEnable(final boolean startDetectorEnable);
|
|||
|
|
++
|
|||
|
|
++ /**
|
|||
|
|
++ * Use it to judge if the detector enabled.
|
|||
|
|
++ *
|
|||
|
|
++ * @return is the detector should be started.
|
|||
|
|
++ */
|
|||
|
|
++ boolean isStartDetectorEnable();
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java
|
|||
|
|
+index 8af629574..d3ff7eb45 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java
|
|||
|
|
+@@ -37,6 +37,8 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance<String>
|
|||
|
|
+ private int detectTimeout = 200;
|
|||
|
|
+ private int detectInterval = 2000;
|
|||
|
|
+ private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex();
|
|||
|
|
++
|
|||
|
|
++ private volatile boolean startDetectorEnable = false;
|
|||
|
|
+ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
|
|||
|
|
+ @Override
|
|||
|
|
+ public Thread newThread(Runnable r) {
|
|||
|
|
+@@ -80,7 +82,9 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance<String>
|
|||
|
|
+ @Override
|
|||
|
|
+ public void run() {
|
|||
|
|
+ try {
|
|||
|
|
+- detectByOneRound();
|
|||
|
|
++ if (startDetectorEnable) {
|
|||
|
|
++ detectByOneRound();
|
|||
|
|
++ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ log.warn("Unexpected exception raised while detecting service reachability", e);
|
|||
|
|
+ }
|
|||
|
|
+@@ -137,6 +141,13 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance<String>
|
|||
|
|
+ this.faultItemTable.remove(name);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public boolean isStartDetectorEnable() {
|
|||
|
|
++ return startDetectorEnable;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setStartDetectorEnable(boolean startDetectorEnable) {
|
|||
|
|
++ this.startDetectorEnable = startDetectorEnable;
|
|||
|
|
++ }
|
|||
|
|
+ @Override
|
|||
|
|
+ public String pickOneAtLeast() {
|
|||
|
|
+ final Enumeration<FaultItem> elements = this.faultItemTable.elements();
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java
|
|||
|
|
+index c01490784..69fb533e5 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java
|
|||
|
|
+@@ -24,8 +24,8 @@ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+
|
|||
|
|
+ public class MQFaultStrategy {
|
|||
|
|
+ private LatencyFaultTolerance<String> latencyFaultTolerance;
|
|||
|
|
+- private boolean sendLatencyFaultEnable;
|
|||
|
|
+- private boolean startDetectorEnable;
|
|||
|
|
++ private volatile boolean sendLatencyFaultEnable;
|
|||
|
|
++ private volatile boolean startDetectorEnable;
|
|||
|
|
+ private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L};
|
|||
|
|
+ private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L};
|
|||
|
|
+
|
|||
|
|
+@@ -64,11 +64,11 @@ public class MQFaultStrategy {
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+ public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) {
|
|||
|
|
+- this.setStartDetectorEnable(cc.isStartDetectorEnable());
|
|||
|
|
+- this.setSendLatencyFaultEnable(cc.isSendLatencyEnable());
|
|||
|
|
+ this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector);
|
|||
|
|
+ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval());
|
|||
|
|
+ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout());
|
|||
|
|
++ this.setStartDetectorEnable(cc.isStartDetectorEnable());
|
|||
|
|
++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ // For unit test.
|
|||
|
|
+@@ -123,21 +123,15 @@ public class MQFaultStrategy {
|
|||
|
|
+
|
|||
|
|
+ public void setStartDetectorEnable(boolean startDetectorEnable) {
|
|||
|
|
+ this.startDetectorEnable = startDetectorEnable;
|
|||
|
|
++ this.latencyFaultTolerance.setStartDetectorEnable(startDetectorEnable);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void startDetector() {
|
|||
|
|
+- // user should start the detector
|
|||
|
|
+- // and the thread should not be in running state.
|
|||
|
|
+- if (this.sendLatencyFaultEnable && this.startDetectorEnable) {
|
|||
|
|
+- // start the detector.
|
|||
|
|
+- this.latencyFaultTolerance.startDetector();
|
|||
|
|
+- }
|
|||
|
|
++ this.latencyFaultTolerance.startDetector();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void shutdown() {
|
|||
|
|
+- if (this.sendLatencyFaultEnable && this.startDetectorEnable) {
|
|||
|
|
+- this.latencyFaultTolerance.shutdown();
|
|||
|
|
+- }
|
|||
|
|
++ this.latencyFaultTolerance.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java
|
|||
|
|
+index 8b3c2f7c8..898e529f8 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java
|
|||
|
|
+@@ -26,7 +26,6 @@ public class MessageQueueView {
|
|||
|
|
+ private final MessageQueueSelector readSelector;
|
|||
|
|
+ private final MessageQueueSelector writeSelector;
|
|||
|
|
+ private final TopicRouteWrapper topicRouteWrapper;
|
|||
|
|
+- private MQFaultStrategy mqFaultStrategy;
|
|||
|
|
+
|
|||
|
|
+ public MessageQueueView(String topic, TopicRouteData topicRouteData, MQFaultStrategy mqFaultStrategy) {
|
|||
|
|
+ this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic);
|
|||
|
|
+@@ -67,12 +66,4 @@ public class MessageQueueView {
|
|||
|
|
+ .add("topicRouteWrapper", topicRouteWrapper)
|
|||
|
|
+ .toString();
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
+- public MQFaultStrategy getMQFaultStrategy() {
|
|||
|
|
+- return mqFaultStrategy;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) {
|
|||
|
|
+- this.mqFaultStrategy = mqFaultStrategy;
|
|||
|
|
+- }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
+index 74769a423..caf62a1e0 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
+@@ -127,7 +127,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown {
|
|||
|
|
+ @Override
|
|||
|
|
+ public String resolve(String name) {
|
|||
|
|
+ try {
|
|||
|
|
+- String brokerAddr = getBrokerAddr(null, name);
|
|||
|
|
++ String brokerAddr = getBrokerAddr(ProxyContext.createForInner("MQFaultStrategy"), name);
|
|||
|
|
+ return brokerAddr;
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ return null;
|
|||
|
|
+@@ -175,9 +175,17 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown {
|
|||
|
|
+
|
|||
|
|
+ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation,
|
|||
|
|
+ boolean reachable) {
|
|||
|
|
++ checkSendFaultToleranceEnable();
|
|||
|
|
+ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public void checkSendFaultToleranceEnable() {
|
|||
|
|
++ boolean hotLatencySwitch = ConfigurationManager.getProxyConfig().isSendLatencyEnable();
|
|||
|
|
++ boolean hotDetectorSwitch = ConfigurationManager.getProxyConfig().isStartDetectorEnable();
|
|||
|
|
++ this.mqFaultStrategy.setSendLatencyFaultEnable(hotLatencySwitch);
|
|||
|
|
++ this.mqFaultStrategy.setStartDetectorEnable(hotDetectorSwitch);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public MQFaultStrategy getMqFaultStrategy() {
|
|||
|
|
+ return this.mqFaultStrategy;
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch017-backport-Convergent-thread-pool-creation.patch b/patch017-backport-Convergent-thread-pool-creation.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..92d0bd2e9
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch017-backport-Convergent-thread-pool-creation.patch
|
|||
|
|
@@ -0,0 +1,2243 @@
|
|||
|
|
+From c100d815d754d7cb330bc63e145bafd2d9b59cb1 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: guyinyou <36399867+guyinyou@users.noreply.github.com>
|
|||
|
|
+Date: Mon, 11 Sep 2023 10:13:56 +0800
|
|||
|
|
+Subject: [PATCH 1/6] [ISSUE #7328] Convergent thread pool creation (#7329)
|
|||
|
|
+
|
|||
|
|
+* Convergence thread pool creation to facilitate subsequent iteration management
|
|||
|
|
+
|
|||
|
|
+* Convergence thread pool creation in ThreadPoolMonitor.java
|
|||
|
|
+
|
|||
|
|
+* fix unit test
|
|||
|
|
+
|
|||
|
|
+* Convergence ThreadPool constructor
|
|||
|
|
+
|
|||
|
|
+* Convergence ScheduledThreadPool constructor
|
|||
|
|
+
|
|||
|
|
+* remove unused import
|
|||
|
|
+
|
|||
|
|
+* Convergence ScheduledThreadPool constructor
|
|||
|
|
+
|
|||
|
|
+* remove unused import
|
|||
|
|
+
|
|||
|
|
+---------
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/broker/BrokerController.java | 39 +++++-----
|
|||
|
|
+ .../client/ClientHousekeepingService.java | 4 +-
|
|||
|
|
+ .../DefaultConsumerIdsChangeListener.java | 3 +-
|
|||
|
|
+ .../broker/controller/ReplicasManager.java | 9 +--
|
|||
|
|
+ .../dledger/DLedgerRoleChangeHandler.java | 4 +-
|
|||
|
|
+ .../broker/failover/EscapeBridge.java | 4 +-
|
|||
|
|
+ .../broker/latency/BrokerFastFailure.java | 5 +-
|
|||
|
|
+ .../BrokerFixedThreadPoolExecutor.java | 57 --------------
|
|||
|
|
+ .../broker/latency/FutureTaskExt.java | 39 ----------
|
|||
|
|
+ .../rocketmq/broker/out/BrokerOuterAPI.java | 7 +-
|
|||
|
|
+ .../schedule/ScheduleMessageService.java | 7 +-
|
|||
|
|
+ .../broker/topic/TopicRouteInfoManager.java | 4 +-
|
|||
|
|
+ ...ractTransactionalMessageCheckListener.java | 4 +-
|
|||
|
|
+ .../rocketmq/broker/BrokerControllerTest.java | 2 +-
|
|||
|
|
+ .../broker/latency/BrokerFastFailureTest.java | 1 +
|
|||
|
|
+ .../common/config/AbstractRocksDBStorage.java | 6 +-
|
|||
|
|
+ .../FutureTaskExtThreadPoolExecutor.java | 3 +-
|
|||
|
|
+ .../common/thread/ThreadPoolMonitor.java | 6 +-
|
|||
|
|
+ .../rocketmq/common/utils/ThreadUtils.java | 74 ++++++++++++++++---
|
|||
|
|
+ .../rocketmq/container/BrokerContainer.java | 6 +-
|
|||
|
|
+ .../controller/ControllerManager.java | 14 +---
|
|||
|
|
+ .../controller/impl/DLedgerController.java | 10 +--
|
|||
|
|
+ .../DefaultBrokerHeartbeatManager.java | 3 +-
|
|||
|
|
+ .../rocketmq/namesrv/NamesrvController.java | 22 ++----
|
|||
|
|
+ .../grpc/v2/channel/GrpcChannelManager.java | 6 +-
|
|||
|
|
+ .../remoting/RemotingProtocolServer.java | 4 +-
|
|||
|
|
+ .../proxy/service/ClusterServiceManager.java | 12 +--
|
|||
|
|
+ .../proxy/service/LocalServiceManager.java | 4 +-
|
|||
|
|
+ .../receipt/DefaultReceiptHandleManager.java | 8 +-
|
|||
|
|
+ .../service/route/TopicRouteService.java | 9 +--
|
|||
|
|
+ .../remoting/netty/NettyRemotingClient.java | 4 +-
|
|||
|
|
+ .../remoting/netty/NettyRemotingServer.java | 4 +-
|
|||
|
|
+ .../rocketmq/store/DefaultMessageStore.java | 8 +-
|
|||
|
|
+ .../ha/autoswitch/AutoSwitchHAService.java | 38 +++++-----
|
|||
|
|
+ .../rocketmq/store/kv/CompactionStore.java | 21 +++---
|
|||
|
|
+ .../store/queue/ConsumeQueueStore.java | 4 +-
|
|||
|
|
+ .../store/stats/BrokerStatsManager.java | 14 ++--
|
|||
|
|
+ .../store/timer/TimerMessageStore.java | 6 +-
|
|||
|
|
+ .../apache/rocketmq/test/util/StatUtil.java | 1 -
|
|||
|
|
+ .../common/TieredStoreExecutor.java | 14 ++--
|
|||
|
|
+ .../tools/admin/DefaultMQAdminExtImpl.java | 3 +-
|
|||
|
|
+ 41 files changed, 215 insertions(+), 278 deletions(-)
|
|||
|
|
+ delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java
|
|||
|
|
+ delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+index 6aba70cb2..275b64b1a 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+@@ -34,7 +34,6 @@ import org.apache.rocketmq.broker.failover.EscapeBridge;
|
|||
|
|
+ import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap;
|
|||
|
|
+ import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
|
|||
|
|
+ import org.apache.rocketmq.broker.latency.BrokerFastFailure;
|
|||
|
|
+-import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor;
|
|||
|
|
+ import org.apache.rocketmq.broker.longpolling.LmqPullRequestHoldService;
|
|||
|
|
+ import org.apache.rocketmq.broker.longpolling.NotifyMessageArrivingListener;
|
|||
|
|
+ import org.apache.rocketmq.broker.longpolling.PullRequestHoldService;
|
|||
|
|
+@@ -98,6 +97,7 @@ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
+ import org.apache.rocketmq.common.stats.MomentStatsItem;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.ServiceProvider;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.Configuration;
|
|||
|
|
+@@ -160,7 +160,6 @@ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ScheduledFuture;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.locks.Lock;
|
|||
|
|
+ import java.util.concurrent.locks.ReentrantLock;
|
|||
|
|
+@@ -455,10 +454,10 @@ public class BrokerController {
|
|||
|
|
+ * Initialize resources including remoting server and thread executors.
|
|||
|
|
+ */
|
|||
|
|
+ protected void initializeResources() {
|
|||
|
|
+- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new ThreadFactoryImpl("BrokerControllerScheduledThread", true, getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.sendMessageExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getSendMessageThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getSendMessageThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -466,7 +465,7 @@ public class BrokerController {
|
|||
|
|
+ this.sendThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.pullMessageExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getPullMessageThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getPullMessageThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -474,7 +473,7 @@ public class BrokerController {
|
|||
|
|
+ this.pullThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("PullMessageThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.litePullMessageExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.litePullMessageExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getLitePullMessageThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getLitePullMessageThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -482,7 +481,7 @@ public class BrokerController {
|
|||
|
|
+ this.litePullThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("LitePullMessageThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.putMessageFutureExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.putMessageFutureExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getPutMessageFutureThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getPutMessageFutureThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -490,7 +489,7 @@ public class BrokerController {
|
|||
|
|
+ this.putThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.ackMessageExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.ackMessageExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getAckMessageThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getAckMessageThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -498,7 +497,7 @@ public class BrokerController {
|
|||
|
|
+ this.ackThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("AckMessageThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.queryMessageExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getQueryMessageThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getQueryMessageThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -506,7 +505,7 @@ public class BrokerController {
|
|||
|
|
+ this.queryThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("QueryMessageThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.adminBrokerExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.adminBrokerExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getAdminBrokerThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getAdminBrokerThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -514,7 +513,7 @@ public class BrokerController {
|
|||
|
|
+ this.adminBrokerThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("AdminBrokerThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.clientManageExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.clientManageExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getClientManageThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getClientManageThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -522,7 +521,7 @@ public class BrokerController {
|
|||
|
|
+ this.clientManagerThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("ClientManageThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.heartbeatExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.heartbeatExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getHeartbeatThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getHeartbeatThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -530,7 +529,7 @@ public class BrokerController {
|
|||
|
|
+ this.heartbeatThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("HeartbeatThread_", true, getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.consumerManageExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.consumerManageExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getConsumerManageThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getConsumerManageThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -538,7 +537,7 @@ public class BrokerController {
|
|||
|
|
+ this.consumerManagerThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("ConsumerManageThread_", true, getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.replyMessageExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getProcessReplyMessageThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getProcessReplyMessageThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -546,7 +545,7 @@ public class BrokerController {
|
|||
|
|
+ this.replyThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("ProcessReplyMessageThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.endTransactionExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.endTransactionExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getEndTransactionThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getEndTransactionThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -554,7 +553,7 @@ public class BrokerController {
|
|||
|
|
+ this.endTransactionThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("EndTransactionThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.loadBalanceExecutor = new BrokerFixedThreadPoolExecutor(
|
|||
|
|
++ this.loadBalanceExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(),
|
|||
|
|
+ this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -562,9 +561,9 @@ public class BrokerController {
|
|||
|
|
+ this.loadBalanceThreadPoolQueue,
|
|||
|
|
+ new ThreadFactoryImpl("LoadBalanceProcessorThread_", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+- this.syncBrokerMemberGroupExecutorService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ this.syncBrokerMemberGroupExecutorService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new ThreadFactoryImpl("BrokerControllerSyncBrokerScheduledThread", getBrokerIdentity()));
|
|||
|
|
+- this.brokerHeartbeatExecutorService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ this.brokerHeartbeatExecutorService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new ThreadFactoryImpl("BrokerControllerHeartbeatScheduledThread", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+ this.topicQueueMappingCleanService = new TopicQueueMappingCleanService(this);
|
|||
|
|
+@@ -828,8 +827,6 @@ public class BrokerController {
|
|||
|
|
+
|
|||
|
|
+ initializeResources();
|
|||
|
|
+
|
|||
|
|
+- registerProcessor();
|
|||
|
|
+-
|
|||
|
|
+ initializeScheduledTasks();
|
|||
|
|
+
|
|||
|
|
+ initialTransaction();
|
|||
|
|
+@@ -1690,6 +1687,8 @@ public class BrokerController {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }, 10, 5, TimeUnit.SECONDS);
|
|||
|
|
++
|
|||
|
|
++ registerProcessor();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ protected void scheduleSendHeartbeat() {
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java
|
|||
|
|
+index 98e5f450f..cbb81f632 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java
|
|||
|
|
+@@ -18,11 +18,11 @@ package org.apache.rocketmq.broker.client;
|
|||
|
|
+
|
|||
|
|
+ import io.netty.channel.Channel;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.ChannelEventListener;
|
|||
|
|
+@@ -35,7 +35,7 @@ public class ClientHousekeepingService implements ChannelEventListener {
|
|||
|
|
+
|
|||
|
|
+ public ClientHousekeepingService(final BrokerController brokerController) {
|
|||
|
|
+ this.brokerController = brokerController;
|
|||
|
|
+- scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new ThreadFactoryImpl("ClientHousekeepingScheduledThread", brokerController.getBrokerIdentity()));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
|
|||
|
|
+index 2ce036a0f..d17a2a547 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
|
|||
|
|
+@@ -22,7 +22,6 @@ import java.util.List;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+ import org.apache.rocketmq.common.AbstractBrokerRunnable;
|
|||
|
|
+@@ -37,7 +36,7 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen
|
|||
|
|
+ private final BrokerController brokerController;
|
|||
|
|
+ private final int cacheSize = 8096;
|
|||
|
|
+
|
|||
|
|
+- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ ThreadUtils.newGenericThreadFactory("DefaultConsumerIdsChangeListener", true));
|
|||
|
|
+
|
|||
|
|
+ private ConcurrentHashMap<String,List<Channel>> consumerChannelMap = new ConcurrentHashMap<>(cacheSize);
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
+index 37c82e434..a989e6e68 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
|
|||
|
|
+@@ -27,10 +27,8 @@ import java.util.concurrent.ArrayBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ScheduledFuture;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+@@ -42,6 +40,7 @@ import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.EpochEntry;
|
|||
|
|
+@@ -107,9 +106,9 @@ public class ReplicasManager {
|
|||
|
|
+ public ReplicasManager(final BrokerController brokerController) {
|
|||
|
|
+ this.brokerController = brokerController;
|
|||
|
|
+ this.brokerOuterAPI = brokerController.getBrokerOuterAPI();
|
|||
|
|
+- this.scheduledService = Executors.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity()));
|
|||
|
|
+- this.executorService = Executors.newFixedThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity()));
|
|||
|
|
+- this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS,
|
|||
|
|
++ this.scheduledService = ThreadUtils.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity()));
|
|||
|
|
++ this.executorService = ThreadUtils.newThreadPoolExecutor(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity()));
|
|||
|
|
++ this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS,
|
|||
|
|
+ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("ReplicasManager_scan_thread_", brokerController.getBrokerIdentity()));
|
|||
|
|
+ this.haService = (AutoSwitchHAService) brokerController.getMessageStore().getHaService();
|
|||
|
|
+ this.brokerConfig = brokerController.getBrokerConfig();
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java
|
|||
|
|
+index 75023ee1b..e6cb97640 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java
|
|||
|
|
+@@ -21,12 +21,12 @@ import io.openmessaging.storage.dledger.DLedgerServer;
|
|||
|
|
+ import io.openmessaging.storage.dledger.MemberState;
|
|||
|
|
+ import io.openmessaging.storage.dledger.utils.DLedgerUtils;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.Future;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
+@@ -49,7 +49,7 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange
|
|||
|
|
+ this.messageStore = messageStore;
|
|||
|
|
+ this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();
|
|||
|
|
+ this.dLegerServer = dLedgerCommitLog.getdLedgerServer();
|
|||
|
|
+- this.executorService = Executors.newSingleThreadExecutor(
|
|||
|
|
++ this.executorService = ThreadUtils.newSingleThreadExecutor(
|
|||
|
|
+ new ThreadFactoryImpl("DLegerRoleChangeHandler_", brokerController.getBrokerIdentity()));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java
|
|||
|
|
+index 7c350fc1d..6a0817480 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java
|
|||
|
|
+@@ -24,7 +24,6 @@ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+@@ -43,6 +42,7 @@ import org.apache.rocketmq.common.message.MessageDecoder;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingException;
|
|||
|
|
+@@ -72,7 +72,7 @@ public class EscapeBridge {
|
|||
|
|
+ public void start() throws Exception {
|
|||
|
|
+ if (brokerController.getBrokerConfig().isEnableSlaveActingMaster() && brokerController.getBrokerConfig().isEnableRemoteEscape()) {
|
|||
|
|
+ final BlockingQueue<Runnable> asyncSenderThreadPoolQueue = new LinkedBlockingQueue<>(50000);
|
|||
|
|
+- this.defaultAsyncSenderExecutor = new ThreadPoolExecutor(
|
|||
|
|
++ this.defaultAsyncSenderExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ Runtime.getRuntime().availableProcessors(),
|
|||
|
|
+ Runtime.getRuntime().availableProcessors(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java
|
|||
|
|
+index d3d0bc8ba..3b6e9dc67 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java
|
|||
|
|
+@@ -18,13 +18,14 @@ package org.apache.rocketmq.broker.latency;
|
|||
|
|
+
|
|||
|
|
+ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+ import org.apache.rocketmq.common.AbstractBrokerRunnable;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.future.FutureTaskExt;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.RequestTask;
|
|||
|
|
+@@ -43,7 +44,7 @@ public class BrokerFastFailure {
|
|||
|
|
+
|
|||
|
|
+ public BrokerFastFailure(final BrokerController brokerController) {
|
|||
|
|
+ this.brokerController = brokerController;
|
|||
|
|
+- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new ThreadFactoryImpl("BrokerFastFailureScheduledThread", true,
|
|||
|
|
+ brokerController == null ? null : brokerController.getBrokerConfig()));
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java
|
|||
|
|
+deleted file mode 100644
|
|||
|
|
+index d2d1143a3..000000000
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java
|
|||
|
|
++++ /dev/null
|
|||
|
|
+@@ -1,57 +0,0 @@
|
|||
|
|
+-/*
|
|||
|
|
+- * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+- * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+- * this work for additional information regarding copyright ownership.
|
|||
|
|
+- * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+- * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+- * the License. You may obtain a copy of the License at
|
|||
|
|
+- *
|
|||
|
|
+- * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+- *
|
|||
|
|
+- * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+- * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+- * See the License for the specific language governing permissions and
|
|||
|
|
+- * limitations under the License.
|
|||
|
|
+- */
|
|||
|
|
+-
|
|||
|
|
+-package org.apache.rocketmq.broker.latency;
|
|||
|
|
+-
|
|||
|
|
+-import java.util.concurrent.BlockingQueue;
|
|||
|
|
+-import java.util.concurrent.RejectedExecutionHandler;
|
|||
|
|
+-import java.util.concurrent.RunnableFuture;
|
|||
|
|
+-import java.util.concurrent.ThreadFactory;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+-import java.util.concurrent.TimeUnit;
|
|||
|
|
+-
|
|||
|
|
+-public class BrokerFixedThreadPoolExecutor extends ThreadPoolExecutor {
|
|||
|
|
+- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime,
|
|||
|
|
+- final TimeUnit unit,
|
|||
|
|
+- final BlockingQueue<Runnable> workQueue) {
|
|||
|
|
+- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue);
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime,
|
|||
|
|
+- final TimeUnit unit,
|
|||
|
|
+- final BlockingQueue<Runnable> workQueue, final ThreadFactory threadFactory) {
|
|||
|
|
+- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory);
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime,
|
|||
|
|
+- final TimeUnit unit,
|
|||
|
|
+- final BlockingQueue<Runnable> workQueue, final RejectedExecutionHandler handler) {
|
|||
|
|
+- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler);
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime,
|
|||
|
|
+- final TimeUnit unit,
|
|||
|
|
+- final BlockingQueue<Runnable> workQueue, final ThreadFactory threadFactory,
|
|||
|
|
+- final RejectedExecutionHandler handler) {
|
|||
|
|
+- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- @Override
|
|||
|
|
+- protected <T> RunnableFuture<T> newTaskFor(final Runnable runnable, final T value) {
|
|||
|
|
+- return new FutureTaskExt<>(runnable, value);
|
|||
|
|
+- }
|
|||
|
|
+-}
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java
|
|||
|
|
+deleted file mode 100644
|
|||
|
|
+index f132efaeb..000000000
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java
|
|||
|
|
++++ /dev/null
|
|||
|
|
+@@ -1,39 +0,0 @@
|
|||
|
|
+-/*
|
|||
|
|
+- * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+- * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+- * this work for additional information regarding copyright ownership.
|
|||
|
|
+- * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+- * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+- * the License. You may obtain a copy of the License at
|
|||
|
|
+- *
|
|||
|
|
+- * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+- *
|
|||
|
|
+- * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+- * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+- * See the License for the specific language governing permissions and
|
|||
|
|
+- * limitations under the License.
|
|||
|
|
+- */
|
|||
|
|
+-
|
|||
|
|
+-package org.apache.rocketmq.broker.latency;
|
|||
|
|
+-
|
|||
|
|
+-import java.util.concurrent.Callable;
|
|||
|
|
+-import java.util.concurrent.FutureTask;
|
|||
|
|
+-
|
|||
|
|
+-public class FutureTaskExt<V> extends FutureTask<V> {
|
|||
|
|
+- private final Runnable runnable;
|
|||
|
|
+-
|
|||
|
|
+- public FutureTaskExt(final Callable<V> callable) {
|
|||
|
|
+- super(callable);
|
|||
|
|
+- this.runnable = null;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- public FutureTaskExt(final Runnable runnable, final V result) {
|
|||
|
|
+- super(runnable, result);
|
|||
|
|
+- this.runnable = runnable;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- public Runnable getRunnable() {
|
|||
|
|
+- return runnable;
|
|||
|
|
+- }
|
|||
|
|
+-}
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
|
|||
|
|
+index ae81e8b11..9dfb8127d 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
|
|||
|
|
+@@ -27,9 +27,9 @@ import java.util.concurrent.ArrayBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.CopyOnWriteArrayList;
|
|||
|
|
+ import java.util.concurrent.CountDownLatch;
|
|||
|
|
++import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+-import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.PullResult;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.PullStatus;
|
|||
|
|
+ import org.apache.rocketmq.client.exception.MQBrokerException;
|
|||
|
|
+@@ -59,6 +59,7 @@ import org.apache.rocketmq.common.namesrv.DefaultTopAddressing;
|
|||
|
|
+ import org.apache.rocketmq.common.namesrv.TopAddressing;
|
|||
|
|
+ import org.apache.rocketmq.common.sysflag.PullSysFlag;
|
|||
|
|
+ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.InvokeCallback;
|
|||
|
|
+@@ -144,7 +145,7 @@ public class BrokerOuterAPI {
|
|||
|
|
+ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
|
|||
|
|
+ private final RemotingClient remotingClient;
|
|||
|
|
+ private final TopAddressing topAddressing = new DefaultTopAddressing(MixAll.getWSAddr());
|
|||
|
|
+- private final BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES,
|
|||
|
|
++ private final ExecutorService brokerOuterExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES,
|
|||
|
|
+ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true));
|
|||
|
|
+ private final ClientMetadata clientMetadata;
|
|||
|
|
+ private final RpcClient rpcClient;
|
|||
|
|
+@@ -1092,7 +1093,7 @@ public class BrokerOuterAPI {
|
|||
|
|
+ throw new MQBrokerException(response.getCode(), response.getRemark());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public BrokerFixedThreadPoolExecutor getBrokerOuterExecutor() {
|
|||
|
|
++ public ExecutorService getBrokerOuterExecutor() {
|
|||
|
|
+ return brokerOuterExecutor;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
|
|||
|
|
+index 297b14207..0c2e6507b 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
|
|||
|
|
+@@ -26,7 +26,6 @@ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.ExecutionException;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicBoolean;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
+@@ -91,7 +90,7 @@ public class ScheduleMessageService extends ConfigManager {
|
|||
|
|
+ public ScheduleMessageService(final BrokerController brokerController) {
|
|||
|
|
+ this.brokerController = brokerController;
|
|||
|
|
+ this.enableAsyncDeliver = brokerController.getMessageStoreConfig().isEnableScheduleAsyncDeliver();
|
|||
|
|
+- scheduledPersistService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ scheduledPersistService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new ThreadFactoryImpl("ScheduleMessageServicePersistThread", true, brokerController.getBrokerConfig()));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -134,9 +133,9 @@ public class ScheduleMessageService extends ConfigManager {
|
|||
|
|
+ public void start() {
|
|||
|
|
+ if (started.compareAndSet(false, true)) {
|
|||
|
|
+ this.load();
|
|||
|
|
+- this.deliverExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_"));
|
|||
|
|
++ this.deliverExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_"));
|
|||
|
|
+ if (this.enableAsyncDeliver) {
|
|||
|
|
+- this.handleExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_"));
|
|||
|
|
++ this.handleExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_"));
|
|||
|
|
+ }
|
|||
|
|
+ for (Map.Entry<Integer, Long> entry : this.delayLevelTable.entrySet()) {
|
|||
|
|
+ Integer level = entry.getKey();
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java
|
|||
|
|
+index b35564725..11bde5f5f 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java
|
|||
|
|
+@@ -23,7 +23,6 @@ import java.util.Objects;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.locks.Lock;
|
|||
|
|
+@@ -36,6 +35,7 @@ import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingException;
|
|||
|
|
+@@ -66,7 +66,7 @@ public class TopicRouteInfoManager {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void start() {
|
|||
|
|
+- this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread"));
|
|||
|
|
++ this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread"));
|
|||
|
|
+
|
|||
|
|
+ this.scheduledExecutorService.scheduleAtFixedRate(() -> {
|
|||
|
|
+ try {
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java
|
|||
|
|
+index 771d84300..982355d78 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java
|
|||
|
|
+@@ -19,7 +19,6 @@ package org.apache.rocketmq.broker.transaction;
|
|||
|
|
+ import io.netty.channel.Channel;
|
|||
|
|
+ import java.util.concurrent.ArrayBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+@@ -27,6 +26,7 @@ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageConst;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;
|
|||
|
|
+@@ -97,7 +97,7 @@ public abstract class AbstractTransactionalMessageCheckListener {
|
|||
|
|
+
|
|||
|
|
+ public synchronized void initExecutorService() {
|
|||
|
|
+ if (executorService == null) {
|
|||
|
|
+- executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000),
|
|||
|
|
++ executorService = ThreadUtils.newThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000),
|
|||
|
|
+ new ThreadFactoryImpl("Transaction-msg-check-thread", brokerController.getBrokerIdentity()), new CallerRunsPolicy());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java
|
|||
|
|
+index 75ad961ce..6035a20ac 100644
|
|||
|
|
+--- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java
|
|||
|
|
++++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java
|
|||
|
|
+@@ -23,9 +23,9 @@ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+
|
|||
|
|
+-import org.apache.rocketmq.broker.latency.FutureTaskExt;
|
|||
|
|
+ import org.apache.rocketmq.common.BrokerConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
++import org.apache.rocketmq.common.future.FutureTaskExt;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyClientConfig;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.RequestTask;
|
|||
|
|
+diff --git a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java
|
|||
|
|
+index 5d0f7f9d7..31b547cf1 100644
|
|||
|
|
+--- a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java
|
|||
|
|
++++ b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java
|
|||
|
|
+@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.latency;
|
|||
|
|
+ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
++import org.apache.rocketmq.common.future.FutureTaskExt;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.RequestTask;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
|
|||
|
|
+index a720a5be3..6f19a9815 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
|
|||
|
|
+@@ -23,7 +23,6 @@ import java.util.List;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+ import java.util.concurrent.ArrayBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.Semaphore;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+@@ -33,6 +32,7 @@ import com.google.common.collect.Maps;
|
|||
|
|
+
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.rocksdb.ColumnFamilyDescriptor;
|
|||
|
|
+@@ -82,8 +82,8 @@ public abstract class AbstractRocksDBStorage {
|
|||
|
|
+ private volatile boolean closed;
|
|||
|
|
+
|
|||
|
|
+ private final Semaphore reloadPermit = new Semaphore(1);
|
|||
|
|
+- private final ScheduledExecutorService reloadScheduler = new ScheduledThreadPoolExecutor(1, new ThreadFactoryImpl("RocksDBStorageReloadService_"));
|
|||
|
|
+- private final ThreadPoolExecutor manualCompactionThread = new ThreadPoolExecutor(
|
|||
|
|
++ private final ScheduledExecutorService reloadScheduler = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("RocksDBStorageReloadService_"));
|
|||
|
|
++ private final ThreadPoolExecutor manualCompactionThread = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ 1, 1, 1000 * 60, TimeUnit.MILLISECONDS,
|
|||
|
|
+ new ArrayBlockingQueue(1),
|
|||
|
|
+ new ThreadFactoryImpl("RocksDBManualCompactionService_"),
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java
|
|||
|
|
+index 411da9221..7b68873a9 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java
|
|||
|
|
+@@ -29,7 +29,8 @@ public class FutureTaskExtThreadPoolExecutor extends ThreadPoolExecutor {
|
|||
|
|
+
|
|||
|
|
+ public FutureTaskExtThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime,
|
|||
|
|
+ TimeUnit unit,
|
|||
|
|
+- BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory,
|
|||
|
|
++ BlockingQueue<Runnable> workQueue,
|
|||
|
|
++ ThreadFactory threadFactory,
|
|||
|
|
+ RejectedExecutionHandler handler) {
|
|||
|
|
+ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java
|
|||
|
|
+index 49d97a5d7..1bfabbffe 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java
|
|||
|
|
+@@ -22,12 +22,12 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
|||
|
|
+ import java.util.Collections;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.concurrent.CopyOnWriteArrayList;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+
|
|||
|
|
+@@ -36,7 +36,7 @@ public class ThreadPoolMonitor {
|
|||
|
|
+ private static Logger waterMarkLogger = LoggerFactory.getLogger(ThreadPoolMonitor.class);
|
|||
|
|
+
|
|||
|
|
+ private static final List<ThreadPoolWrapper> MONITOR_EXECUTOR = new CopyOnWriteArrayList<>();
|
|||
|
|
+- private static final ScheduledExecutorService MONITOR_SCHEDULED = Executors.newSingleThreadScheduledExecutor(
|
|||
|
|
++ private static final ScheduledExecutorService MONITOR_SCHEDULED = ThreadUtils.newSingleThreadScheduledExecutor(
|
|||
|
|
+ new ThreadFactoryBuilder().setNameFormat("ThreadPoolMonitor-%d").build()
|
|||
|
|
+ );
|
|||
|
|
+
|
|||
|
|
+@@ -81,7 +81,7 @@ public class ThreadPoolMonitor {
|
|||
|
|
+ String name,
|
|||
|
|
+ int queueCapacity,
|
|||
|
|
+ List<ThreadPoolStatusMonitor> threadPoolStatusMonitors) {
|
|||
|
|
+- ThreadPoolExecutor executor = new FutureTaskExtThreadPoolExecutor(
|
|||
|
|
++ ThreadPoolExecutor executor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ corePoolSize,
|
|||
|
|
+ maximumPoolSize,
|
|||
|
|
+ keepAliveTime,
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java
|
|||
|
|
+index 4b366d4e3..1644c6360 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java
|
|||
|
|
+@@ -20,38 +20,94 @@ package org.apache.rocketmq.common.utils;
|
|||
|
|
+ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.Executors;
|
|||
|
|
++import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
++import java.util.concurrent.RejectedExecutionHandler;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
++import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.ThreadFactory;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.thread.FutureTaskExtThreadPoolExecutor;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+
|
|||
|
|
+ public final class ThreadUtils {
|
|||
|
|
+ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
+- public static ExecutorService newThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime,
|
|||
|
|
+- TimeUnit unit, BlockingQueue<Runnable> workQueue, String processName, boolean isDaemon) {
|
|||
|
|
+- return new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon));
|
|||
|
|
++ public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) {
|
|||
|
|
++ return ThreadUtils.newSingleThreadExecutor(newThreadFactory(processName, isDaemon));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) {
|
|||
|
|
+- return Executors.newSingleThreadExecutor(newThreadFactory(processName, isDaemon));
|
|||
|
|
++ public static ExecutorService newSingleThreadExecutor(ThreadFactory threadFactory) {
|
|||
|
|
++ return ThreadUtils.newThreadPoolExecutor(1, threadFactory);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory) {
|
|||
|
|
++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, corePoolSize,
|
|||
|
|
++ 0L, TimeUnit.MILLISECONDS,
|
|||
|
|
++ new LinkedBlockingQueue<>(),
|
|||
|
|
++ threadFactory);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public static ExecutorService newThreadPoolExecutor(int corePoolSize,
|
|||
|
|
++ int maximumPoolSize,
|
|||
|
|
++ long keepAliveTime,
|
|||
|
|
++ TimeUnit unit, BlockingQueue<Runnable> workQueue,
|
|||
|
|
++ String processName,
|
|||
|
|
++ boolean isDaemon) {
|
|||
|
|
++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public static ExecutorService newThreadPoolExecutor(final int corePoolSize,
|
|||
|
|
++ final int maximumPoolSize,
|
|||
|
|
++ final long keepAliveTime,
|
|||
|
|
++ final TimeUnit unit,
|
|||
|
|
++ final BlockingQueue<Runnable> workQueue,
|
|||
|
|
++ final ThreadFactory threadFactory) {
|
|||
|
|
++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, new ThreadPoolExecutor.AbortPolicy());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public static ExecutorService newThreadPoolExecutor(int corePoolSize,
|
|||
|
|
++ int maximumPoolSize,
|
|||
|
|
++ long keepAliveTime,
|
|||
|
|
++ TimeUnit unit,
|
|||
|
|
++ BlockingQueue<Runnable> workQueue,
|
|||
|
|
++ ThreadFactory threadFactory,
|
|||
|
|
++ RejectedExecutionHandler handler) {
|
|||
|
|
++ return new FutureTaskExtThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static ScheduledExecutorService newSingleThreadScheduledExecutor(String processName, boolean isDaemon) {
|
|||
|
|
+- return Executors.newSingleThreadScheduledExecutor(newThreadFactory(processName, isDaemon));
|
|||
|
|
++ return ThreadUtils.newScheduledThreadPool(1, processName, isDaemon);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public static ScheduledExecutorService newSingleThreadScheduledExecutor(ThreadFactory threadFactory) {
|
|||
|
|
++ return ThreadUtils.newScheduledThreadPool(1, threadFactory);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize) {
|
|||
|
|
++ return ThreadUtils.newScheduledThreadPool(corePoolSize, Executors.defaultThreadFactory());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public static ScheduledExecutorService newFixedThreadScheduledPool(int nThreads, String processName,
|
|||
|
|
++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, String processName,
|
|||
|
|
+ boolean isDaemon) {
|
|||
|
|
+- return Executors.newScheduledThreadPool(nThreads, newThreadFactory(processName, isDaemon));
|
|||
|
|
++ return ThreadUtils.newScheduledThreadPool(corePoolSize, newThreadFactory(processName, isDaemon));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, ThreadFactory threadFactory) {
|
|||
|
|
++ return ThreadUtils.newScheduledThreadPool(corePoolSize, threadFactory, new ThreadPoolExecutor.AbortPolicy());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize,
|
|||
|
|
++ ThreadFactory threadFactory,
|
|||
|
|
++ RejectedExecutionHandler handler) {
|
|||
|
|
++ return new ScheduledThreadPoolExecutor(corePoolSize, threadFactory, handler);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static ThreadFactory newThreadFactory(String processName, boolean isDaemon) {
|
|||
|
|
+- return newGenericThreadFactory("Remoting-" + processName, isDaemon);
|
|||
|
|
++ return newGenericThreadFactory("ThreadUtils-" + processName, isDaemon);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static ThreadFactory newGenericThreadFactory(String processName) {
|
|||
|
|
+diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java
|
|||
|
|
+index c6446f058..5b712bc30 100644
|
|||
|
|
+--- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java
|
|||
|
|
++++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java
|
|||
|
|
+@@ -47,14 +47,12 @@ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+
|
|||
|
|
+ public class BrokerContainer implements IBrokerContainer {
|
|||
|
|
+ private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
+- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new BasicThreadFactory.Builder()
|
|||
|
|
+ .namingPattern("BrokerContainerScheduledThread")
|
|||
|
|
+ .daemon(true)
|
|||
|
|
+@@ -143,7 +141,7 @@ public class BrokerContainer implements IBrokerContainer {
|
|||
|
|
+ this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.containerClientHouseKeepingService);
|
|||
|
|
+ this.fastRemotingServer = this.remotingServer.newRemotingServer(this.nettyServerConfig.getListenPort() - 2);
|
|||
|
|
+
|
|||
|
|
+- this.brokerContainerExecutor = new ThreadPoolExecutor(
|
|||
|
|
++ this.brokerContainerExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ 1,
|
|||
|
|
+ 1,
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java
|
|||
|
|
+index 7c91e70da..3e6b0eba5 100644
|
|||
|
|
+--- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java
|
|||
|
|
++++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java
|
|||
|
|
+@@ -25,8 +25,6 @@ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.Future;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+-import java.util.concurrent.RunnableFuture;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+@@ -34,8 +32,8 @@ import org.apache.rocketmq.common.ControllerConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+-import org.apache.rocketmq.common.future.FutureTaskExt;
|
|||
|
|
+
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy;
|
|||
|
|
+ import org.apache.rocketmq.controller.impl.DLedgerController;
|
|||
|
|
+ import org.apache.rocketmq.controller.impl.heartbeat.DefaultBrokerHeartbeatManager;
|
|||
|
|
+@@ -93,18 +91,14 @@ public class ControllerManager {
|
|||
|
|
+
|
|||
|
|
+ public boolean initialize() {
|
|||
|
|
+ this.controllerRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.controllerConfig.getControllerRequestThreadPoolQueueCapacity());
|
|||
|
|
+- this.controllerRequestExecutor = new ThreadPoolExecutor(
|
|||
|
|
++ this.controllerRequestExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.controllerConfig.getControllerThreadPoolNums(),
|
|||
|
|
+ this.controllerConfig.getControllerThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+ TimeUnit.MILLISECONDS,
|
|||
|
|
+ this.controllerRequestThreadPoolQueue,
|
|||
|
|
+- new ThreadFactoryImpl("ControllerRequestExecutorThread_")) {
|
|||
|
|
+- @Override
|
|||
|
|
+- protected <T> RunnableFuture<T> newTaskFor(final Runnable runnable, final T value) {
|
|||
|
|
+- return new FutureTaskExt<T>(runnable, value);
|
|||
|
|
+- }
|
|||
|
|
+- };
|
|||
|
|
++ new ThreadFactoryImpl("ControllerRequestExecutorThread_"));
|
|||
|
|
++
|
|||
|
|
+ this.notifyService.initialize();
|
|||
|
|
+ if (StringUtils.isEmpty(this.controllerConfig.getControllerDLegerPeers())) {
|
|||
|
|
+ throw new IllegalArgumentException("Attribute value controllerDLegerPeers of ControllerConfig is null or empty");
|
|||
|
|
+diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java
|
|||
|
|
+index fa91f288e..33e4406e4 100644
|
|||
|
|
+--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java
|
|||
|
|
++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java
|
|||
|
|
+@@ -32,7 +32,6 @@ import java.util.Map;
|
|||
|
|
+ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ScheduledFuture;
|
|||
|
|
+@@ -44,6 +43,7 @@ import org.apache.rocketmq.common.ControllerConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.ServiceThread;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.controller.Controller;
|
|||
|
|
+ import org.apache.rocketmq.controller.elect.ElectPolicy;
|
|||
|
|
+ import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy;
|
|||
|
|
+@@ -66,11 +66,11 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.ResponseCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.SyncStateSet;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;
|
|||
|
|
+@@ -136,7 +136,7 @@ public class DLedgerController implements Controller {
|
|||
|
|
+ this.dLedgerServer = new DLedgerServer(dLedgerConfig, nettyServerConfig, nettyClientConfig, channelEventListener);
|
|||
|
|
+ this.dLedgerServer.registerStateMachine(this.statemachine);
|
|||
|
|
+ this.dLedgerServer.getDLedgerLeaderElector().addRoleChangeHandler(this.roleHandler);
|
|||
|
|
+- this.scanInactiveMasterService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_"));
|
|||
|
|
++ this.scanInactiveMasterService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_"));
|
|||
|
|
+ this.brokerLifecycleListeners = new ArrayList<>();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -513,7 +513,7 @@ public class DLedgerController implements Controller {
|
|||
|
|
+ class RoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler {
|
|||
|
|
+
|
|||
|
|
+ private final String selfId;
|
|||
|
|
+- private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_"));
|
|||
|
|
++ private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_"));
|
|||
|
|
+ private volatile MemberState.Role currentRole = MemberState.Role.FOLLOWER;
|
|||
|
|
+
|
|||
|
|
+ public RoleChangeHandler(final String selfId) {
|
|||
|
|
+diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java
|
|||
|
|
+index 2fbddb9cd..6ebb2c994 100644
|
|||
|
|
+--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java
|
|||
|
|
++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java
|
|||
|
|
+@@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.common.ControllerConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.controller.BrokerHeartbeatManager;
|
|||
|
|
+ import org.apache.rocketmq.controller.helper.BrokerLifecycleListener;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+@@ -66,7 +67,7 @@ public class DefaultBrokerHeartbeatManager implements BrokerHeartbeatManager {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void initialize() {
|
|||
|
|
+- this.scheduledService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_"));
|
|||
|
|
++ this.scheduledService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_"));
|
|||
|
|
+ this.executor = Executors.newFixedThreadPool(2, new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_executorService_"));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java
|
|||
|
|
+index 15c65ebec..be327cffa 100644
|
|||
|
|
+--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java
|
|||
|
|
++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java
|
|||
|
|
+@@ -20,10 +20,7 @@ import java.util.Collections;
|
|||
|
|
+ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+-import java.util.concurrent.RunnableFuture;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.commons.lang3.concurrent.BasicThreadFactory;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+@@ -31,6 +28,7 @@ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.future.FutureTaskExt;
|
|||
|
|
+ import org.apache.rocketmq.common.namesrv.NamesrvConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.NetworkUtil;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.namesrv.kvconfig.KVConfigManager;
|
|||
|
|
+@@ -62,10 +60,10 @@ public class NamesrvController {
|
|||
|
|
+ private final NettyServerConfig nettyServerConfig;
|
|||
|
|
+ private final NettyClientConfig nettyClientConfig;
|
|||
|
|
+
|
|||
|
|
+- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new BasicThreadFactory.Builder().namingPattern("NSScheduledThread").daemon(true).build());
|
|||
|
|
+
|
|||
|
|
+- private final ScheduledExecutorService scanExecutorService = new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ private final ScheduledExecutorService scanExecutorService = ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new BasicThreadFactory.Builder().namingPattern("NSScanScheduledThread").daemon(true).build());
|
|||
|
|
+
|
|||
|
|
+ private final KVConfigManager kvConfigManager;
|
|||
|
|
+@@ -138,20 +136,10 @@ public class NamesrvController {
|
|||
|
|
+
|
|||
|
|
+ private void initiateThreadExecutors() {
|
|||
|
|
+ this.defaultThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getDefaultThreadPoolQueueCapacity());
|
|||
|
|
+- this.defaultExecutor = new ThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_")) {
|
|||
|
|
+- @Override
|
|||
|
|
+- protected <T> RunnableFuture<T> newTaskFor(final Runnable runnable, final T value) {
|
|||
|
|
+- return new FutureTaskExt<>(runnable, value);
|
|||
|
|
+- }
|
|||
|
|
+- };
|
|||
|
|
++ this.defaultExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_"));
|
|||
|
|
+
|
|||
|
|
+ this.clientRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getClientRequestThreadPoolQueueCapacity());
|
|||
|
|
+- this.clientRequestExecutor = new ThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_")) {
|
|||
|
|
+- @Override
|
|||
|
|
+- protected <T> RunnableFuture<T> newTaskFor(final Runnable runnable, final T value) {
|
|||
|
|
+- return new FutureTaskExt<>(runnable, value);
|
|||
|
|
+- }
|
|||
|
|
+- };
|
|||
|
|
++ this.clientRequestExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_"));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void initiateSslContext() {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java
|
|||
|
|
+index 14330dd8d..a18cf7600 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java
|
|||
|
|
+@@ -21,13 +21,13 @@ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicLong;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+-import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.StartAndShutdown;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ProxyConfig;
|
|||
|
|
+ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
|
|||
|
|
+@@ -43,7 +43,7 @@ public class GrpcChannelManager implements StartAndShutdown {
|
|||
|
|
+ protected final AtomicLong nonceIdGenerator = new AtomicLong(0);
|
|||
|
|
+ protected final ConcurrentMap<String /* nonce */, ResultFuture> resultNonceFutureMap = new ConcurrentHashMap<>();
|
|||
|
|
+
|
|||
|
|
+- protected final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(
|
|||
|
|
++ protected final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(
|
|||
|
|
+ new ThreadFactoryImpl("GrpcChannelManager_")
|
|||
|
|
+ );
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
|
|||
|
|
+index bcc9edd09..fe07090d5 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
|
|||
|
|
+@@ -22,7 +22,6 @@ import io.netty.channel.Channel;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+@@ -33,6 +32,7 @@ import org.apache.rocketmq.common.future.FutureTaskExt;
|
|||
|
|
+ import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
|
|||
|
|
+ import org.apache.rocketmq.common.thread.ThreadPoolStatusMonitor;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.StartAndShutdown;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+@@ -178,7 +178,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
|
|||
|
|
+ new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInDefaultQueue())
|
|||
|
|
+ );
|
|||
|
|
+
|
|||
|
|
+- this.timerExecutor = Executors.newSingleThreadScheduledExecutor(
|
|||
|
|
++ this.timerExecutor = ThreadUtils.newSingleThreadScheduledExecutor(
|
|||
|
|
+ new ThreadFactoryBuilder().setNameFormat("RemotingServerScheduler-%d").build()
|
|||
|
|
+ );
|
|||
|
|
+ this.timerExecutor.scheduleAtFixedRate(this::cleanExpireRequest, 10, 10, TimeUnit.SECONDS);
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
|
|||
|
|
+index d2ddfc352..9786cec55 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
|
|||
|
|
+@@ -16,7 +16,6 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.proxy.service;
|
|||
|
|
+
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.broker.client.ClientChannelInfo;
|
|||
|
|
+@@ -27,23 +26,24 @@ import org.apache.rocketmq.broker.client.ProducerChangeListener;
|
|||
|
|
+ import org.apache.rocketmq.broker.client.ProducerGroupEvent;
|
|||
|
|
+ import org.apache.rocketmq.broker.client.ProducerManager;
|
|||
|
|
+ import org.apache.rocketmq.client.common.NameserverAccessConfig;
|
|||
|
|
++import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor;
|
|||
|
|
++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.AbstractStartAndShutdown;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+-import org.apache.rocketmq.common.utils.AbstractStartAndShutdown;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ProxyConfig;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.admin.AdminService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.admin.DefaultAdminService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.client.ClusterConsumerManager;
|
|||
|
|
++import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.message.ClusterMessageService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.message.MessageService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.metadata.ClusterMetadataService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.metadata.MetadataService;
|
|||
|
|
+-import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor;
|
|||
|
|
+-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+-import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.relay.ClusterProxyRelayService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.route.ClusterTopicRouteService;
|
|||
|
|
+@@ -73,7 +73,7 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
|
|||
|
|
+ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
|
|||
|
|
+ NameserverAccessConfig nameserverAccessConfig = new NameserverAccessConfig(proxyConfig.getNamesrvAddr(),
|
|||
|
|
+ proxyConfig.getNamesrvDomain(), proxyConfig.getNamesrvDomainSubgroup());
|
|||
|
|
+- this.scheduledExecutorService = Executors.newScheduledThreadPool(3);
|
|||
|
|
++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(3);
|
|||
|
|
+
|
|||
|
|
+ this.messagingClientAPIFactory = new MQClientAPIFactory(
|
|||
|
|
+ nameserverAccessConfig,
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java
|
|||
|
|
+index 4d1ca7b66..59cd92685 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java
|
|||
|
|
+@@ -16,7 +16,6 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.proxy.service;
|
|||
|
|
+
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+@@ -28,6 +27,7 @@ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.StartAndShutdown;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ProxyConfig;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.admin.AdminService;
|
|||
|
|
+@@ -58,7 +58,7 @@ public class LocalServiceManager extends AbstractStartAndShutdown implements Ser
|
|||
|
|
+ private final MQClientAPIFactory mqClientAPIFactory;
|
|||
|
|
+ private final ChannelManager channelManager;
|
|||
|
|
+
|
|||
|
|
+- private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(
|
|||
|
|
++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(
|
|||
|
|
+ new ThreadFactoryImpl("LocalServiceManagerScheduledThread"));
|
|||
|
|
+
|
|||
|
|
+ public LocalServiceManager(BrokerController brokerController, RPCHook rpcHook) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java
|
|||
|
|
+index 69f44344a..207603fe8 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java
|
|||
|
|
+@@ -24,7 +24,6 @@ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+@@ -42,20 +41,21 @@ import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.StartAndShutdown;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+-import org.apache.rocketmq.proxy.common.RenewEvent;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.MessageReceiptHandle;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyContext;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyException;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.ReceiptHandleGroup;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.RenewEvent;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.RenewStrategyPolicy;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.ExceptionUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ProxyConfig;
|
|||
|
|
+-import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey;
|
|||
|
|
+ import org.apache.rocketmq.proxy.service.metadata.MetadataService;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.subscription.RetryPolicy;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
|
|||
|
|
+@@ -68,7 +68,7 @@ public class DefaultReceiptHandleManager extends AbstractStartAndShutdown implem
|
|||
|
|
+ protected final StateEventListener<RenewEvent> eventListener;
|
|||
|
|
+ protected final static RetryPolicy RENEW_POLICY = new RenewStrategyPolicy();
|
|||
|
|
+ protected final ScheduledExecutorService scheduledExecutorService =
|
|||
|
|
+- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_"));
|
|||
|
|
++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_"));
|
|||
|
|
+ protected final ThreadPoolExecutor renewalWorkerService;
|
|||
|
|
+
|
|||
|
|
+ public DefaultReceiptHandleManager(MetadataService metadataService, ConsumerManager consumerManager, StateEventListener<RenewEvent> eventListener) {
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
+index caf62a1e0..ccf094c03 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
|
|||
|
|
+@@ -19,25 +19,24 @@ package org.apache.rocketmq.proxy.service.route;
|
|||
|
|
+ import com.github.benmanes.caffeine.cache.CacheLoader;
|
|||
|
|
+ import com.github.benmanes.caffeine.cache.Caffeine;
|
|||
|
|
+ import com.github.benmanes.caffeine.cache.LoadingCache;
|
|||
|
|
++import com.google.common.base.Optional;
|
|||
|
|
+ import java.time.Duration;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+-
|
|||
|
|
+-import com.google.common.base.Optional;
|
|||
|
|
+ import org.apache.rocketmq.client.ClientConfig;
|
|||
|
|
+ import org.apache.rocketmq.client.exception.MQClientException;
|
|||
|
|
++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+ import org.apache.rocketmq.client.latency.MQFaultStrategy;
|
|||
|
|
+ import org.apache.rocketmq.client.latency.Resolver;
|
|||
|
|
+ import org.apache.rocketmq.client.latency.ServiceDetector;
|
|||
|
|
+-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.Address;
|
|||
|
|
+@@ -63,7 +62,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown {
|
|||
|
|
+ public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) {
|
|||
|
|
+ ProxyConfig config = ConfigurationManager.getProxyConfig();
|
|||
|
|
+
|
|||
|
|
+- this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(
|
|||
|
|
++ this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(
|
|||
|
|
+ new ThreadFactoryImpl("TopicRouteService_")
|
|||
|
|
+ );
|
|||
|
|
+ this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor(
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
+index 8491f4354..64621dd6c 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
+@@ -61,7 +61,6 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.Executors;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicReference;
|
|||
|
|
+@@ -71,6 +70,7 @@ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.ChannelEventListener;
|
|||
|
|
+@@ -142,7 +142,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+
|
|||
|
|
+ this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactoryImpl("NettyClientPublicExecutor_"));
|
|||
|
|
+
|
|||
|
|
+- this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS,
|
|||
|
|
++ this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS,
|
|||
|
|
+ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("NettyClientScan_thread_"));
|
|||
|
|
+
|
|||
|
|
+ if (eventLoopGroup != null) {
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
+index e626260c9..aa0d46542 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
+@@ -61,6 +61,7 @@ import org.apache.rocketmq.common.constant.HAProxyConstants;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.BinaryUtil;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.NetworkUtil;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.ChannelEventListener;
|
|||
|
|
+@@ -83,7 +84,6 @@ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+
|
|||
|
|
+@@ -171,7 +171,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ScheduledExecutorService buildScheduleExecutor() {
|
|||
|
|
+- return new ScheduledThreadPoolExecutor(1,
|
|||
|
|
++ return ThreadUtils.newScheduledThreadPool(1,
|
|||
|
|
+ new ThreadFactoryImpl("NettyServerScheduler_", true),
|
|||
|
|
+ new ThreadPoolExecutor.DiscardOldestPolicy());
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
+index f2a54ddf6..02ea47f13 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
+@@ -48,7 +48,6 @@ import java.util.concurrent.ConcurrentLinkedQueue;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.ExecutionException;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+@@ -83,6 +82,7 @@ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.CleanupPolicyUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.QueueTypeUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.ServiceProvider;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
|
|||
|
|
+@@ -205,7 +205,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
+ private ConcurrentMap<String, TopicConfig> topicConfigTable;
|
|||
|
|
+
|
|||
|
|
+ private final ScheduledExecutorService scheduledCleanQueueExecutorService =
|
|||
|
|
+- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread"));
|
|||
|
|
++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread"));
|
|||
|
|
+
|
|||
|
|
+ public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
|
|||
|
|
+ final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap<String, TopicConfig> topicConfigTable) throws IOException {
|
|||
|
|
+@@ -253,7 +253,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
+ this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog());
|
|||
|
|
+
|
|||
|
|
+ this.scheduledExecutorService =
|
|||
|
|
+- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity()));
|
|||
|
|
++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity()));
|
|||
|
|
+
|
|||
|
|
+ this.dispatcherList = new LinkedList<>();
|
|||
|
|
+ this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue());
|
|||
|
|
+@@ -2915,7 +2915,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
+ private final ExecutorService batchDispatchRequestExecutor;
|
|||
|
|
+
|
|||
|
|
+ public MainBatchDispatchRequestService() {
|
|||
|
|
+- batchDispatchRequestExecutor = new ThreadPoolExecutor(
|
|||
|
|
++ batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
|
|||
|
|
+ DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
+index d5393fdca..f20bc3e28 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
+@@ -17,10 +17,26 @@
|
|||
|
|
+
|
|||
|
|
+ package org.apache.rocketmq.store.ha.autoswitch;
|
|||
|
|
+
|
|||
|
|
+-
|
|||
|
|
++import java.io.IOException;
|
|||
|
|
++import java.nio.channels.SocketChannel;
|
|||
|
|
++import java.util.ArrayList;
|
|||
|
|
++import java.util.HashSet;
|
|||
|
|
++import java.util.Iterator;
|
|||
|
|
++import java.util.List;
|
|||
|
|
++import java.util.Map;
|
|||
|
|
++import java.util.Objects;
|
|||
|
|
++import java.util.Set;
|
|||
|
|
++import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
++import java.util.concurrent.ExecutorService;
|
|||
|
|
++import java.util.concurrent.locks.Lock;
|
|||
|
|
++import java.util.concurrent.locks.ReadWriteLock;
|
|||
|
|
++import java.util.concurrent.locks.ReentrantReadWriteLock;
|
|||
|
|
++import java.util.function.Consumer;
|
|||
|
|
++import java.util.stream.Collectors;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.EpochEntry;
|
|||
|
|
+@@ -36,30 +52,12 @@ import org.apache.rocketmq.store.ha.HAClient;
|
|||
|
|
+ import org.apache.rocketmq.store.ha.HAConnection;
|
|||
|
|
+ import org.apache.rocketmq.store.ha.HAConnectionStateNotificationService;
|
|||
|
|
+
|
|||
|
|
+-import java.io.IOException;
|
|||
|
|
+-import java.nio.channels.SocketChannel;
|
|||
|
|
+-import java.util.ArrayList;
|
|||
|
|
+-import java.util.HashSet;
|
|||
|
|
+-import java.util.List;
|
|||
|
|
+-import java.util.Iterator;
|
|||
|
|
+-import java.util.Map;
|
|||
|
|
+-import java.util.Objects;
|
|||
|
|
+-import java.util.Set;
|
|||
|
|
+-import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+-import java.util.concurrent.ExecutorService;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+-import java.util.concurrent.locks.Lock;
|
|||
|
|
+-import java.util.concurrent.locks.ReadWriteLock;
|
|||
|
|
+-import java.util.concurrent.locks.ReentrantReadWriteLock;
|
|||
|
|
+-import java.util.function.Consumer;
|
|||
|
|
+-import java.util.stream.Collectors;
|
|||
|
|
+-
|
|||
|
|
+ /**
|
|||
|
|
+ * SwitchAble ha service, support switch role to master or slave.
|
|||
|
|
+ */
|
|||
|
|
+ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
+ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
|
|||
|
|
+- private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_"));
|
|||
|
|
++ private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_"));
|
|||
|
|
+ private final ConcurrentHashMap<Long/*brokerId*/, Long/*lastCaughtUpTimestamp*/> connectionCaughtUpTimeTable = new ConcurrentHashMap<>();
|
|||
|
|
+ private final List<Consumer<Set<Long/*brokerId*/>>> syncStateSetChangedListeners = new ArrayList<>();
|
|||
|
|
+ private final Set<Long/*brokerId*/> syncStateSet = new HashSet<>();
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java
|
|||
|
|
+index b37c90726..639084fa2 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java
|
|||
|
|
+@@ -16,17 +16,25 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.store.kv;
|
|||
|
|
+
|
|||
|
|
+-import java.util.Random;
|
|||
|
|
++import java.io.File;
|
|||
|
|
++import java.io.IOException;
|
|||
|
|
++import java.nio.file.Files;
|
|||
|
|
++import java.nio.file.Paths;
|
|||
|
|
+ import java.util.Iterator;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+ import java.util.Objects;
|
|||
|
|
+ import java.util.Optional;
|
|||
|
|
++import java.util.Random;
|
|||
|
|
++import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
++import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
++import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
+ import org.apache.rocketmq.common.attribute.CleanupPolicy;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.CleanupPolicyUtils;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
+@@ -35,15 +43,6 @@ import org.apache.rocketmq.store.GetMessageResult;
|
|||
|
|
+ import org.apache.rocketmq.store.SelectMappedBufferResult;
|
|||
|
|
+ import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
+
|
|||
|
|
+-import java.io.File;
|
|||
|
|
+-import java.io.IOException;
|
|||
|
|
+-import java.nio.file.Files;
|
|||
|
|
+-import java.nio.file.Paths;
|
|||
|
|
+-import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+-import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.TimeUnit;
|
|||
|
|
+-
|
|||
|
|
+ public class CompactionStore {
|
|||
|
|
+
|
|||
|
|
+ public static final String COMPACTION_DIR = "compaction";
|
|||
|
|
+@@ -76,7 +75,7 @@ public class CompactionStore {
|
|||
|
|
+ this.positionMgr = new CompactionPositionMgr(compactionPath);
|
|||
|
|
+ this.compactionThreadNum = Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, config.getCompactionThreadNum()));
|
|||
|
|
+
|
|||
|
|
+- this.compactionSchedule = Executors.newScheduledThreadPool(this.compactionThreadNum,
|
|||
|
|
++ this.compactionSchedule = ThreadUtils.newScheduledThreadPool(this.compactionThreadNum,
|
|||
|
|
+ new ThreadFactoryImpl("compactionSchedule_"));
|
|||
|
|
+ this.offsetMapSize = config.getMaxOffsetMapSize() / compactionThreadNum;
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
|
|||
|
|
+index 8d38503b3..d03d15d65 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
|
|||
|
|
+@@ -24,7 +24,6 @@ import java.util.concurrent.CountDownLatch;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.FutureTask;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
+@@ -34,6 +33,7 @@ import org.apache.rocketmq.common.message.MessageDecoder;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.QueueTypeUtils;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.store.CommitLog;
|
|||
|
|
+@@ -175,7 +175,7 @@ public class ConsumeQueueStore {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ExecutorService buildExecutorService(BlockingQueue<Runnable> blockingQueue, String threadNamePrefix) {
|
|||
|
|
+- return new ThreadPoolExecutor(
|
|||
|
|
++ return ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(),
|
|||
|
|
+ this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
|
|||
|
|
+index 2dd3fc5b5..489d7b4fb 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
|
|||
|
|
+@@ -17,7 +17,6 @@
|
|||
|
|
+ package org.apache.rocketmq.store.stats;
|
|||
|
|
+
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import org.apache.commons.lang3.tuple.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.BrokerConfig;
|
|||
|
|
+@@ -32,13 +31,14 @@ import org.apache.rocketmq.common.statistics.StatisticsItemScheduledPrinter;
|
|||
|
|
+ import org.apache.rocketmq.common.statistics.StatisticsItemStateGetter;
|
|||
|
|
+ import org.apache.rocketmq.common.statistics.StatisticsKindMeta;
|
|||
|
|
+ import org.apache.rocketmq.common.statistics.StatisticsManager;
|
|||
|
|
++import org.apache.rocketmq.common.stats.MomentStatsItemSet;
|
|||
|
|
+ import org.apache.rocketmq.common.stats.Stats;
|
|||
|
|
++import org.apache.rocketmq.common.stats.StatsItem;
|
|||
|
|
++import org.apache.rocketmq.common.stats.StatsItemSet;
|
|||
|
|
+ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+-import org.apache.rocketmq.common.stats.MomentStatsItemSet;
|
|||
|
|
+-import org.apache.rocketmq.common.stats.StatsItem;
|
|||
|
|
+-import org.apache.rocketmq.common.stats.StatsItemSet;
|
|||
|
|
+
|
|||
|
|
+ public class BrokerStatsManager {
|
|||
|
|
+
|
|||
|
|
+@@ -281,11 +281,11 @@ public class BrokerStatsManager {
|
|||
|
|
+
|
|||
|
|
+ private void initScheduleService() {
|
|||
|
|
+ this.scheduledExecutorService =
|
|||
|
|
+- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig));
|
|||
|
|
++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig));
|
|||
|
|
+ this.commercialExecutor =
|
|||
|
|
+- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig));
|
|||
|
|
++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig));
|
|||
|
|
+ this.accountExecutor =
|
|||
|
|
+- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig));
|
|||
|
|
++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public MomentStatsItemSet getMomentStatsItemSetFallSize() {
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
+index 181f7087a..0d50de65a 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
+@@ -35,7 +35,6 @@ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ConcurrentSkipListSet;
|
|||
|
|
+ import java.util.concurrent.CountDownLatch;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingDeque;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+@@ -54,6 +53,7 @@ import org.apache.rocketmq.common.message.MessageDecoder;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
+ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.store.ConsumeQueue;
|
|||
|
|
+@@ -174,11 +174,11 @@ public class TimerMessageStore {
|
|||
|
|
+ this.lastBrokerRole = storeConfig.getBrokerRole();
|
|||
|
|
+
|
|||
|
|
+ if (messageStore instanceof DefaultMessageStore) {
|
|||
|
|
+- scheduler = Executors.newSingleThreadScheduledExecutor(
|
|||
|
|
++ scheduler = ThreadUtils.newSingleThreadScheduledExecutor(
|
|||
|
|
+ new ThreadFactoryImpl("TimerScheduledThread",
|
|||
|
|
+ ((DefaultMessageStore) messageStore).getBrokerIdentity()));
|
|||
|
|
+ } else {
|
|||
|
|
+- scheduler = Executors.newSingleThreadScheduledExecutor(
|
|||
|
|
++ scheduler = ThreadUtils.newSingleThreadScheduledExecutor(
|
|||
|
|
+ new ThreadFactoryImpl("TimerScheduledThread"));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java
|
|||
|
|
+index f3d105bc6..080b7e385 100644
|
|||
|
|
+--- a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java
|
|||
|
|
++++ b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java
|
|||
|
|
+@@ -28,7 +28,6 @@ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicLong;
|
|||
|
|
+-
|
|||
|
|
+ import javax.annotation.Generated;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java
|
|||
|
|
+index 6dd0e8846..65d586f43 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java
|
|||
|
|
+@@ -20,10 +20,10 @@ import java.util.concurrent.BlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ScheduledThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+
|
|||
|
|
+ public class TieredStoreExecutor {
|
|||
|
|
+
|
|||
|
|
+@@ -43,20 +43,20 @@ public class TieredStoreExecutor {
|
|||
|
|
+ public static ExecutorService compactIndexFileExecutor;
|
|||
|
|
+
|
|||
|
|
+ public static void init() {
|
|||
|
|
+- commonScheduledExecutor = new ScheduledThreadPoolExecutor(
|
|||
|
|
++ commonScheduledExecutor = ThreadUtils.newScheduledThreadPool(
|
|||
|
|
+ Math.max(4, Runtime.getRuntime().availableProcessors()),
|
|||
|
|
+ new ThreadFactoryImpl("TieredCommonExecutor_"));
|
|||
|
|
+
|
|||
|
|
+- commitExecutor = new ScheduledThreadPoolExecutor(
|
|||
|
|
++ commitExecutor = ThreadUtils.newScheduledThreadPool(
|
|||
|
|
+ Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
|
|||
|
|
+ new ThreadFactoryImpl("TieredCommitExecutor_"));
|
|||
|
|
+
|
|||
|
|
+- cleanExpiredFileExecutor = new ScheduledThreadPoolExecutor(
|
|||
|
|
++ cleanExpiredFileExecutor = ThreadUtils.newScheduledThreadPool(
|
|||
|
|
+ Math.max(4, Runtime.getRuntime().availableProcessors()),
|
|||
|
|
+ new ThreadFactoryImpl("TieredCleanFileExecutor_"));
|
|||
|
|
+
|
|||
|
|
+ dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
|
|||
|
|
+- dispatchExecutor = new ThreadPoolExecutor(
|
|||
|
|
++ dispatchExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ Math.max(2, Runtime.getRuntime().availableProcessors()),
|
|||
|
|
+ Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -66,7 +66,7 @@ public class TieredStoreExecutor {
|
|||
|
|
+ new ThreadPoolExecutor.DiscardOldestPolicy());
|
|||
|
|
+
|
|||
|
|
+ fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
|
|||
|
|
+- fetchDataExecutor = new ThreadPoolExecutor(
|
|||
|
|
++ fetchDataExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
|
|||
|
|
+ Math.max(64, Runtime.getRuntime().availableProcessors() * 8),
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+@@ -75,7 +75,7 @@ public class TieredStoreExecutor {
|
|||
|
|
+ new ThreadFactoryImpl("TieredFetchExecutor_"));
|
|||
|
|
+
|
|||
|
|
+ compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
|
|||
|
|
+- compactIndexFileExecutor = new ThreadPoolExecutor(
|
|||
|
|
++ compactIndexFileExecutor = ThreadUtils.newThreadPoolExecutor(
|
|||
|
|
+ 1,
|
|||
|
|
+ 1,
|
|||
|
|
+ 1000 * 60,
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
|
|||
|
|
+index fa3596d51..1ebff6d8a 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
|
|||
|
|
+@@ -66,6 +66,7 @@ import org.apache.rocketmq.common.namesrv.NamesrvUtil;
|
|||
|
|
+ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.NetworkUtil;
|
|||
|
|
+ import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
++import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+@@ -193,7 +194,7 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner {
|
|||
|
|
+
|
|||
|
|
+ int threadPoolCoreSize = Integer.parseInt(System.getProperty("rocketmq.admin.threadpool.coresize", "20"));
|
|||
|
|
+
|
|||
|
|
+- this.threadPoolExecutor = new ThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_"));
|
|||
|
|
++ this.threadPoolExecutor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_"));
|
|||
|
|
+
|
|||
|
|
+ break;
|
|||
|
|
+ case RUNNING:
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From dad6b4dadfec7a58e78a6715ec16c2eb6b17ff27 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ziyi Tan <ajb459684460@gmail.com>
|
|||
|
|
+Date: Mon, 11 Sep 2023 14:34:10 +0800
|
|||
|
|
+Subject: [PATCH 2/6] [ISSUE #7334] `registerIncrementBrokerData` for single
|
|||
|
|
+ topic update (#7335)
|
|||
|
|
+
|
|||
|
|
+Signed-off-by: Ziy1-Tan <ajb459684460@gmail.com>
|
|||
|
|
+---
|
|||
|
|
+ .../broker/topic/TopicConfigManager.java | 30 +++++++++++++++----
|
|||
|
|
+ 1 file changed, 25 insertions(+), 5 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
+index 4e3c1736c..754605438 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
+@@ -290,7 +290,11 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (createNew) {
|
|||
|
|
+- this.brokerController.registerBrokerAll(false, true, true);
|
|||
|
|
++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
++ this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
++ } else {
|
|||
|
|
++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return topicConfig;
|
|||
|
|
+@@ -394,7 +398,11 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (createNew) {
|
|||
|
|
+- this.brokerController.registerBrokerAll(false, true, true);
|
|||
|
|
++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
++ this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
++ } else {
|
|||
|
|
++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return topicConfig;
|
|||
|
|
+@@ -435,7 +443,11 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (createNew) {
|
|||
|
|
+- this.brokerController.registerBrokerAll(false, true, true);
|
|||
|
|
++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
++ this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
++ } else {
|
|||
|
|
++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return topicConfig;
|
|||
|
|
+@@ -461,7 +473,11 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ dataVersion.nextVersion(stateMachineVersion);
|
|||
|
|
+
|
|||
|
|
+ this.persist();
|
|||
|
|
+- this.brokerController.registerBrokerAll(false, true, true);
|
|||
|
|
++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
++ this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
++ } else {
|
|||
|
|
++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -484,7 +500,11 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ dataVersion.nextVersion(stateMachineVersion);
|
|||
|
|
+
|
|||
|
|
+ this.persist();
|
|||
|
|
+- this.brokerController.registerBrokerAll(false, true, true);
|
|||
|
|
++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
++ this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
++ } else {
|
|||
|
|
++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 0dbd0772b99f618f757d42cd64542b83e2100e4f Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ziyi Tan <ajb459684460@gmail.com>
|
|||
|
|
+Date: Mon, 11 Sep 2023 15:48:07 +0800
|
|||
|
|
+Subject: [PATCH 3/6] [ISSUE #7326] Split the request to register to the
|
|||
|
|
+ nameserver (#7325)
|
|||
|
|
+
|
|||
|
|
+Signed-off-by: Ziy1-Tan <ajb459684460@gmail.com>
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/broker/BrokerController.java | 41 +++++++++++--------
|
|||
|
|
+ .../broker/topic/TopicConfigManager.java | 21 ++++++++++
|
|||
|
|
+ .../apache/rocketmq/common/BrokerConfig.java | 24 +++++++++++
|
|||
|
|
+ .../test/route/CreateAndUpdateTopicIT.java | 31 ++++++++++++++
|
|||
|
|
+ 4 files changed, 99 insertions(+), 18 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+index 275b64b1a..9e49f636d 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+@@ -1765,29 +1765,34 @@ public class BrokerController {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) {
|
|||
|
|
++ ConcurrentMap<String, TopicConfig> topicConfigMap = this.getTopicConfigManager().getTopicConfigTable();
|
|||
|
|
++ ConcurrentHashMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();
|
|||
|
|
+
|
|||
|
|
+- TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper();
|
|||
|
|
+-
|
|||
|
|
+- topicConfigWrapper.setDataVersion(this.getTopicConfigManager().getDataVersion());
|
|||
|
|
+- topicConfigWrapper.setTopicConfigTable(this.getTopicConfigManager().getTopicConfigTable());
|
|||
|
|
+-
|
|||
|
|
+- topicConfigWrapper.setTopicQueueMappingInfoMap(this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream().map(
|
|||
|
|
+- entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue()))
|
|||
|
|
+- ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
|
|||
|
|
+-
|
|||
|
|
+- if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())
|
|||
|
|
+- || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {
|
|||
|
|
+- ConcurrentHashMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();
|
|||
|
|
+- for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) {
|
|||
|
|
+- TopicConfig tmp =
|
|||
|
|
++ for (TopicConfig topicConfig : topicConfigMap.values()) {
|
|||
|
|
++ if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())
|
|||
|
|
++ || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {
|
|||
|
|
++ topicConfigTable.put(topicConfig.getTopicName(),
|
|||
|
|
+ new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(),
|
|||
|
|
+- topicConfig.getPerm() & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag());
|
|||
|
|
+- topicConfigTable.put(topicConfig.getTopicName(), tmp);
|
|||
|
|
++ topicConfig.getPerm() & getBrokerConfig().getBrokerPermission()));
|
|||
|
|
++ } else {
|
|||
|
|
++ topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ if (this.brokerConfig.isEnableSplitRegistration()
|
|||
|
|
++ && topicConfigTable.size() >= this.brokerConfig.getSplitRegistrationSize()) {
|
|||
|
|
++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildSerializeWrapper(topicConfigTable);
|
|||
|
|
++ doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper);
|
|||
|
|
++ topicConfigTable.clear();
|
|||
|
|
+ }
|
|||
|
|
+- topicConfigWrapper.setTopicConfigTable(topicConfigTable);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(),
|
|||
|
|
++ Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap = this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream()
|
|||
|
|
++ .map(entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue())))
|
|||
|
|
++ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
|||
|
|
++
|
|||
|
|
++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().
|
|||
|
|
++ buildSerializeWrapper(topicConfigTable, topicQueueMappingInfoMap);
|
|||
|
|
++ if (this.brokerConfig.isEnableSplitRegistration() || forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(),
|
|||
|
|
+ this.getBrokerAddr(),
|
|||
|
|
+ this.brokerConfig.getBrokerName(),
|
|||
|
|
+ this.brokerConfig.getBrokerId(),
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
+index 754605438..8537929be 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
+@@ -29,6 +29,7 @@ import java.util.concurrent.locks.ReentrantLock;
|
|||
|
|
+
|
|||
|
|
+ import com.google.common.collect.ImmutableMap;
|
|||
|
|
+
|
|||
|
|
++import com.google.common.collect.Maps;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerController;
|
|||
|
|
+ import org.apache.rocketmq.broker.BrokerPathConfigHelper;
|
|||
|
|
+@@ -47,7 +48,9 @@ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.DataVersion;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.KVTable;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;
|
|||
|
|
+
|
|||
|
|
+ import static com.google.common.base.Preconditions.checkNotNull;
|
|||
|
|
+
|
|||
|
|
+@@ -609,6 +612,24 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ return topicConfigSerializeWrapper;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper(final ConcurrentMap<String, TopicConfig> topicConfigTable) {
|
|||
|
|
++ return buildSerializeWrapper(topicConfigTable, Maps.newHashMap());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper(
|
|||
|
|
++ final ConcurrentMap<String, TopicConfig> topicConfigTable,
|
|||
|
|
++ final Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap
|
|||
|
|
++ ) {
|
|||
|
|
++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper();
|
|||
|
|
++ topicConfigWrapper.setTopicConfigTable(topicConfigTable);
|
|||
|
|
++ topicConfigWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap);
|
|||
|
|
++ topicConfigWrapper.setDataVersion(this.getDataVersion());
|
|||
|
|
++ if (this.brokerController.getBrokerConfig().isEnableSplitRegistration()) {
|
|||
|
|
++ this.getDataVersion().nextVersion();
|
|||
|
|
++ }
|
|||
|
|
++ return topicConfigWrapper;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public String encode() {
|
|||
|
|
+ return encode(false);
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
|
|||
|
|
+index 45d26b29c..0d248c4e1 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
|
|||
|
|
+@@ -396,6 +396,14 @@ public class BrokerConfig extends BrokerIdentity {
|
|||
|
|
+
|
|||
|
|
+ private boolean enableMixedMessageType = false;
|
|||
|
|
+
|
|||
|
|
++ /**
|
|||
|
|
++ * This flag and deleteTopicWithBrokerRegistration flag in the NameServer cannot be set to true at the same time,
|
|||
|
|
++ * otherwise there will be a loss of routing
|
|||
|
|
++ */
|
|||
|
|
++ private boolean enableSplitRegistration = false;
|
|||
|
|
++
|
|||
|
|
++ private int splitRegistrationSize = 800;
|
|||
|
|
++
|
|||
|
|
+ public long getMaxPopPollingSize() {
|
|||
|
|
+ return maxPopPollingSize;
|
|||
|
|
+ }
|
|||
|
|
+@@ -1731,4 +1739,20 @@ public class BrokerConfig extends BrokerIdentity {
|
|||
|
|
+ public void setEnableMixedMessageType(boolean enableMixedMessageType) {
|
|||
|
|
+ this.enableMixedMessageType = enableMixedMessageType;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isEnableSplitRegistration() {
|
|||
|
|
++ return enableSplitRegistration;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setEnableSplitRegistration(boolean enableSplitRegistration) {
|
|||
|
|
++ this.enableSplitRegistration = enableSplitRegistration;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public int getSplitRegistrationSize() {
|
|||
|
|
++ return splitRegistrationSize;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setSplitRegistrationSize(int splitRegistrationSize) {
|
|||
|
|
++ this.splitRegistrationSize = splitRegistrationSize;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java
|
|||
|
|
+index 7e3c7b871..2370e68c0 100644
|
|||
|
|
+--- a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java
|
|||
|
|
++++ b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java
|
|||
|
|
+@@ -17,6 +17,7 @@
|
|||
|
|
+
|
|||
|
|
+ package org.apache.rocketmq.test.route;
|
|||
|
|
+
|
|||
|
|
++import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
|
|||
|
|
+ import org.apache.rocketmq.test.base.BaseConf;
|
|||
|
|
+ import org.apache.rocketmq.test.util.MQAdminTestUtils;
|
|||
|
|
+@@ -111,4 +112,34 @@ public class CreateAndUpdateTopicIT extends BaseConf {
|
|||
|
|
+ brokerController3.getBrokerConfig().setEnableSingleTopicRegister(false);
|
|||
|
|
+ namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(false);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testCreateOrUpdateTopic_EnableSplitRegistration() {
|
|||
|
|
++ brokerController1.getBrokerConfig().setEnableSplitRegistration(true);
|
|||
|
|
++ brokerController2.getBrokerConfig().setEnableSplitRegistration(true);
|
|||
|
|
++ brokerController3.getBrokerConfig().setEnableSplitRegistration(true);
|
|||
|
|
++
|
|||
|
|
++ String testTopic = "test-topic-";
|
|||
|
|
++
|
|||
|
|
++ for (int i = 0; i < 1000; i++) {
|
|||
|
|
++ TopicConfig topicConfig = new TopicConfig(testTopic + i, 8, 8);
|
|||
|
|
++ brokerController1.getTopicConfigManager().updateTopicConfig(topicConfig);
|
|||
|
|
++ brokerController2.getTopicConfigManager().updateTopicConfig(topicConfig);
|
|||
|
|
++ brokerController3.getTopicConfigManager().updateTopicConfig(topicConfig);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ brokerController1.registerBrokerAll(false, true, true);
|
|||
|
|
++ brokerController2.registerBrokerAll(false, true, true);
|
|||
|
|
++ brokerController3.registerBrokerAll(false, true, true);
|
|||
|
|
++
|
|||
|
|
++ for (int i = 0; i < 1000; i++) {
|
|||
|
|
++ TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic + i);
|
|||
|
|
++ assertThat(route.getBrokerDatas()).hasSize(3);
|
|||
|
|
++ assertThat(route.getQueueDatas()).hasSize(3);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ brokerController1.getBrokerConfig().setEnableSplitRegistration(false);
|
|||
|
|
++ brokerController2.getBrokerConfig().setEnableSplitRegistration(false);
|
|||
|
|
++ brokerController3.getBrokerConfig().setEnableSplitRegistration(false);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From a9e353285cea762b0c5eab567bdfa8e5c8c2d279 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: rongtong <jinrongtong5@163.com>
|
|||
|
|
+Date: Mon, 11 Sep 2023 15:55:18 +0800
|
|||
|
|
+Subject: [PATCH 4/6] Add the configuration of topicQueueLock number to better
|
|||
|
|
+ support different scenarios (#7317)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../main/java/org/apache/rocketmq/store/CommitLog.java | 2 +-
|
|||
|
|
+ .../java/org/apache/rocketmq/store/TopicQueueLock.java | 8 ++++++++
|
|||
|
|
+ .../rocketmq/store/config/MessageStoreConfig.java | 10 ++++++++++
|
|||
|
|
+ 3 files changed, 19 insertions(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
|
|||
|
|
+index e6ee3bacc..456bf2b86 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
|
|||
|
|
+@@ -122,7 +122,7 @@ public class CommitLog implements Swappable {
|
|||
|
|
+
|
|||
|
|
+ this.flushDiskWatcher = new FlushDiskWatcher();
|
|||
|
|
+
|
|||
|
|
+- this.topicQueueLock = new TopicQueueLock();
|
|||
|
|
++ this.topicQueueLock = new TopicQueueLock(messageStore.getMessageStoreConfig().getTopicQueueLockNum());
|
|||
|
|
+
|
|||
|
|
+ this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java
|
|||
|
|
+index a78eeed23..5a131b5c3 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java
|
|||
|
|
+@@ -34,6 +34,14 @@ public class TopicQueueLock {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public TopicQueueLock(int size) {
|
|||
|
|
++ this.size = size;
|
|||
|
|
++ this.lockList = new ArrayList<>(size);
|
|||
|
|
++ for (int i = 0; i < this.size; i++) {
|
|||
|
|
++ this.lockList.add(new ReentrantLock());
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public void lock(String topicQueueKey) {
|
|||
|
|
+ Lock lock = this.lockList.get((topicQueueKey.hashCode() & 0x7fffffff) % this.size);
|
|||
|
|
+ lock.lock();
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
|
|||
|
|
+index efb728ac0..9fa448043 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
|
|||
|
|
+@@ -401,6 +401,8 @@ public class MessageStoreConfig {
|
|||
|
|
+ private long memTableFlushInterval = 60 * 60 * 1000L;
|
|||
|
|
+ private boolean enableRocksDBLog = false;
|
|||
|
|
+
|
|||
|
|
++ private int topicQueueLockNum = 32;
|
|||
|
|
++
|
|||
|
|
+ public boolean isDebugLockEnable() {
|
|||
|
|
+ return debugLockEnable;
|
|||
|
|
+ }
|
|||
|
|
+@@ -1751,4 +1753,12 @@ public class MessageStoreConfig {
|
|||
|
|
+ public void setEnableRocksDBLog(boolean enableRocksDBLog) {
|
|||
|
|
+ this.enableRocksDBLog = enableRocksDBLog;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public int getTopicQueueLockNum() {
|
|||
|
|
++ return topicQueueLockNum;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setTopicQueueLockNum(int topicQueueLockNum) {
|
|||
|
|
++ this.topicQueueLockNum = topicQueueLockNum;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 57f04c95d3a2ba6b91583058a6e4eda209f72d6e Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: guyinyou <36399867+guyinyou@users.noreply.github.com>
|
|||
|
|
+Date: Mon, 11 Sep 2023 18:23:25 +0800
|
|||
|
|
+Subject: [PATCH 5/6] [ISSUE #7343] Rollback modifications to registerProcessor
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: guyinyou <guyinyou.gyy@alibaba-inc.com>
|
|||
|
|
+---
|
|||
|
|
+ .../java/org/apache/rocketmq/broker/BrokerController.java | 4 ++--
|
|||
|
|
+ 1 file changed, 2 insertions(+), 2 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+index 9e49f636d..13a3feb4e 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+@@ -827,6 +827,8 @@ public class BrokerController {
|
|||
|
|
+
|
|||
|
|
+ initializeResources();
|
|||
|
|
+
|
|||
|
|
++ registerProcessor();
|
|||
|
|
++
|
|||
|
|
+ initializeScheduledTasks();
|
|||
|
|
+
|
|||
|
|
+ initialTransaction();
|
|||
|
|
+@@ -1687,8 +1689,6 @@ public class BrokerController {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }, 10, 5, TimeUnit.SECONDS);
|
|||
|
|
+-
|
|||
|
|
+- registerProcessor();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ protected void scheduleSendHeartbeat() {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From dad6ad09d13dadc36b6342671c77f619bbb8c522 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ao Qiao <qiao_ao@foxmail.com>
|
|||
|
|
+Date: Tue, 12 Sep 2023 08:28:45 +0800
|
|||
|
|
+Subject: [PATCH 6/6] [ISSUE #7340] Abstract Duplicate code into a method in
|
|||
|
|
+ `TopicConfigManager` (#7341)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../broker/topic/TopicConfigManager.java | 44 ++++++-------------
|
|||
|
|
+ 1 file changed, 14 insertions(+), 30 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
+index 8537929be..511d29e12 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
|
|||
|
|
+@@ -293,11 +293,7 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (createNew) {
|
|||
|
|
+- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
+- this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
+- } else {
|
|||
|
|
+- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
+- }
|
|||
|
|
++ registerBrokerData(topicConfig);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return topicConfig;
|
|||
|
|
+@@ -337,11 +333,7 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ log.error("createTopicIfAbsent ", e);
|
|||
|
|
+ }
|
|||
|
|
+ if (createNew && register) {
|
|||
|
|
+- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
+- this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
+- } else {
|
|||
|
|
+- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
+- }
|
|||
|
|
++ registerBrokerData(topicConfig);
|
|||
|
|
+ }
|
|||
|
|
+ return getTopicConfig(topicConfig.getTopicName());
|
|||
|
|
+ }
|
|||
|
|
+@@ -401,11 +393,7 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (createNew) {
|
|||
|
|
+- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
+- this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
+- } else {
|
|||
|
|
+- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
+- }
|
|||
|
|
++ registerBrokerData(topicConfig);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return topicConfig;
|
|||
|
|
+@@ -446,11 +434,7 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (createNew) {
|
|||
|
|
+- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
+- this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
+- } else {
|
|||
|
|
+- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
+- }
|
|||
|
|
++ registerBrokerData(topicConfig);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return topicConfig;
|
|||
|
|
+@@ -476,11 +460,7 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ dataVersion.nextVersion(stateMachineVersion);
|
|||
|
|
+
|
|||
|
|
+ this.persist();
|
|||
|
|
+- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
+- this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
+- } else {
|
|||
|
|
+- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
+- }
|
|||
|
|
++ registerBrokerData(topicConfig);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -503,11 +483,7 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ dataVersion.nextVersion(stateMachineVersion);
|
|||
|
|
+
|
|||
|
|
+ this.persist();
|
|||
|
|
+- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
+- this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
+- } else {
|
|||
|
|
+- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
+- }
|
|||
|
|
++ registerBrokerData(topicConfig);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -699,6 +675,14 @@ public class TopicConfigManager extends ConfigManager {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ private void registerBrokerData(TopicConfig topicConfig) {
|
|||
|
|
++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
|
|||
|
|
++ this.brokerController.registerSingleTopicAll(topicConfig);
|
|||
|
|
++ } else {
|
|||
|
|
++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public boolean containsTopic(String topic) {
|
|||
|
|
+ return topicConfigTable.containsKey(topic);
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch018-backport-enhancement-of-tiered-storage.patch b/patch018-backport-enhancement-of-tiered-storage.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..ed79df9c7
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch018-backport-enhancement-of-tiered-storage.patch
|
|||
|
|
@@ -0,0 +1,601 @@
|
|||
|
|
+From 1a8e7cb17cb29ed33b0196b52e452a6e76ade781 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: yuz10 <845238369@qq.com>
|
|||
|
|
+Date: Tue, 12 Sep 2023 19:33:41 +0800
|
|||
|
|
+Subject: [PATCH 1/5] [ISSUE #7345] Fix wrong result of searchOffset in tiered
|
|||
|
|
+ storage
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../tieredstore/file/TieredFlatFile.java | 5 +-
|
|||
|
|
+ .../tieredstore/file/TieredFlatFileTest.java | 46 +++++++++++++++++--
|
|||
|
|
+ 2 files changed, 46 insertions(+), 5 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
|
|||
|
|
+index 426c4e09d..d973179ee 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
|
|||
|
|
+@@ -365,7 +365,10 @@ public class TieredFlatFile {
|
|||
|
|
+ if (!segmentList.isEmpty()) {
|
|||
|
|
+ return boundaryType == BoundaryType.UPPER ? segmentList.get(0) : segmentList.get(segmentList.size() - 1);
|
|||
|
|
+ }
|
|||
|
|
+- return fileSegmentList.isEmpty() ? null : fileSegmentList.get(fileSegmentList.size() - 1);
|
|||
|
|
++ if (fileSegmentList.isEmpty()) {
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++ return boundaryType == BoundaryType.UPPER ? fileSegmentList.get(fileSegmentList.size() - 1) : fileSegmentList.get(0);
|
|||
|
|
+ } finally {
|
|||
|
|
+ fileSegmentLock.readLock().unlock();
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java
|
|||
|
|
+index 7a4d05969..7e2fbf201 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java
|
|||
|
|
+@@ -16,10 +16,7 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.tieredstore.file;
|
|||
|
|
+
|
|||
|
|
+-import java.io.IOException;
|
|||
|
|
+-import java.nio.ByteBuffer;
|
|||
|
|
+-import java.util.ArrayList;
|
|||
|
|
+-import java.util.List;
|
|||
|
|
++import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.FileSegmentType;
|
|||
|
|
+@@ -35,6 +32,11 @@ import org.junit.Assert;
|
|||
|
|
+ import org.junit.Before;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
++import java.io.IOException;
|
|||
|
|
++import java.nio.ByteBuffer;
|
|||
|
|
++import java.util.ArrayList;
|
|||
|
|
++import java.util.List;
|
|||
|
|
++
|
|||
|
|
+ public class TieredFlatFileTest {
|
|||
|
|
+
|
|||
|
|
+ private final String storePath = TieredStoreTestUtil.getRandomStorePath();
|
|||
|
|
+@@ -301,4 +303,40 @@ public class TieredFlatFileTest {
|
|||
|
|
+ fileQueue.rollingNewFile();
|
|||
|
|
+ Assert.assertEquals(2, fileQueue.getFileSegmentCount());
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testGetFileByTime() {
|
|||
|
|
++ String filePath = TieredStoreUtil.toPath(queue);
|
|||
|
|
++ TieredFlatFile tieredFlatFile = fileQueueFactory.createFlatFileForCommitLog(filePath);
|
|||
|
|
++ TieredFileSegment fileSegment1 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig);
|
|||
|
|
++ fileSegment1.setMinTimestamp(100);
|
|||
|
|
++ fileSegment1.setMaxTimestamp(200);
|
|||
|
|
++
|
|||
|
|
++ TieredFileSegment fileSegment2 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig);
|
|||
|
|
++ fileSegment2.setMinTimestamp(200);
|
|||
|
|
++ fileSegment2.setMaxTimestamp(300);
|
|||
|
|
++
|
|||
|
|
++ tieredFlatFile.getFileSegmentList().add(fileSegment1);
|
|||
|
|
++ tieredFlatFile.getFileSegmentList().add(fileSegment2);
|
|||
|
|
++
|
|||
|
|
++ TieredFileSegment segmentUpper = tieredFlatFile.getFileByTime(400, BoundaryType.UPPER);
|
|||
|
|
++ Assert.assertEquals(fileSegment2, segmentUpper);
|
|||
|
|
++
|
|||
|
|
++ TieredFileSegment segmentLower = tieredFlatFile.getFileByTime(400, BoundaryType.LOWER);
|
|||
|
|
++ Assert.assertEquals(fileSegment2, segmentLower);
|
|||
|
|
++
|
|||
|
|
++
|
|||
|
|
++ TieredFileSegment segmentUpper2 = tieredFlatFile.getFileByTime(0, BoundaryType.UPPER);
|
|||
|
|
++ Assert.assertEquals(fileSegment1, segmentUpper2);
|
|||
|
|
++
|
|||
|
|
++ TieredFileSegment segmentLower2 = tieredFlatFile.getFileByTime(0, BoundaryType.LOWER);
|
|||
|
|
++ Assert.assertEquals(fileSegment1, segmentLower2);
|
|||
|
|
++
|
|||
|
|
++
|
|||
|
|
++ TieredFileSegment segmentUpper3 = tieredFlatFile.getFileByTime(200, BoundaryType.UPPER);
|
|||
|
|
++ Assert.assertEquals(fileSegment1, segmentUpper3);
|
|||
|
|
++
|
|||
|
|
++ TieredFileSegment segmentLower3 = tieredFlatFile.getFileByTime(200, BoundaryType.LOWER);
|
|||
|
|
++ Assert.assertEquals(fileSegment2, segmentLower3);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From fd32dae2ab59f86dd215eeec405bf4fa6212bcb3 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lizhimins <707364882@qq.com>
|
|||
|
|
+Date: Tue, 12 Sep 2023 19:58:08 +0800
|
|||
|
|
+Subject: [PATCH 2/5] [ISSUE #6633] Not clear uninitialized files and fix
|
|||
|
|
+ metadata recover (#7342)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../tieredstore/file/TieredFlatFile.java | 53 +++++++------------
|
|||
|
|
+ .../file/TieredFlatFileManager.java | 10 ++--
|
|||
|
|
+ 2 files changed, 22 insertions(+), 41 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
|
|||
|
|
+index d973179ee..d96eb6e8f 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
|
|||
|
|
+@@ -16,7 +16,6 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.tieredstore.file;
|
|||
|
|
+
|
|||
|
|
+-import com.alibaba.fastjson.JSON;
|
|||
|
|
+ import com.google.common.annotations.VisibleForTesting;
|
|||
|
|
+ import java.nio.ByteBuffer;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+@@ -25,13 +24,13 @@ import java.util.Comparator;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+ import java.util.LinkedList;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+-import java.util.Objects;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.CopyOnWriteArrayList;
|
|||
|
|
+ import java.util.concurrent.locks.ReentrantReadWriteLock;
|
|||
|
|
+ import java.util.stream.Collectors;
|
|||
|
|
+ import javax.annotation.Nullable;
|
|||
|
|
++import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.common.AppendResult;
|
|||
|
|
+@@ -43,7 +42,6 @@ import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.provider.FileSegmentAllocator;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
|
|||
|
|
+ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
+-import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+
|
|||
|
|
+ public class TieredFlatFile {
|
|||
|
|
+
|
|||
|
|
+@@ -177,7 +175,10 @@ public class TieredFlatFile {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fileSegment) {
|
|||
|
|
++ /**
|
|||
|
|
++ * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full
|
|||
|
|
++ */
|
|||
|
|
++ public void updateFileSegment(TieredFileSegment fileSegment) {
|
|||
|
|
+
|
|||
|
|
+ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment(
|
|||
|
|
+ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset());
|
|||
|
|
+@@ -186,45 +187,24 @@ public class TieredFlatFile {
|
|||
|
|
+ if (metadata == null) {
|
|||
|
|
+ metadata = new FileSegmentMetadata(
|
|||
|
|
+ this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType());
|
|||
|
|
+- metadata.setCreateTimestamp(fileSegment.getMinTimestamp());
|
|||
|
|
+- metadata.setBeginTimestamp(fileSegment.getMinTimestamp());
|
|||
|
|
+- metadata.setEndTimestamp(fileSegment.getMaxTimestamp());
|
|||
|
|
+- if (fileSegment.isClosed()) {
|
|||
|
|
+- metadata.setStatus(FileSegmentMetadata.STATUS_DELETED);
|
|||
|
|
+- }
|
|||
|
|
+- this.tieredMetadataStore.updateFileSegment(metadata);
|
|||
|
|
++ metadata.setCreateTimestamp(System.currentTimeMillis());
|
|||
|
|
+ }
|
|||
|
|
+- return metadata;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- /**
|
|||
|
|
+- * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full
|
|||
|
|
+- */
|
|||
|
|
+- public void updateFileSegment(TieredFileSegment fileSegment) {
|
|||
|
|
+- FileSegmentMetadata segmentMetadata = getOrCreateFileSegmentMetadata(fileSegment);
|
|||
|
|
+
|
|||
|
|
+- if (segmentMetadata.getStatus() == FileSegmentMetadata.STATUS_NEW
|
|||
|
|
+- && fileSegment.isFull()
|
|||
|
|
+- && !fileSegment.needCommit()) {
|
|||
|
|
++ metadata.setSize(fileSegment.getCommitPosition());
|
|||
|
|
++ metadata.setBeginTimestamp(fileSegment.getMinTimestamp());
|
|||
|
|
++ metadata.setEndTimestamp(fileSegment.getMaxTimestamp());
|
|||
|
|
+
|
|||
|
|
+- segmentMetadata.markSealed();
|
|||
|
|
++ if (fileSegment.isFull() && !fileSegment.needCommit()) {
|
|||
|
|
++ if (metadata.getStatus() == FileSegmentMetadata.STATUS_NEW) {
|
|||
|
|
++ metadata.markSealed();
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (fileSegment.isClosed()) {
|
|||
|
|
+- segmentMetadata.setStatus(FileSegmentMetadata.STATUS_DELETED);
|
|||
|
|
++ metadata.setStatus(FileSegmentMetadata.STATUS_DELETED);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- segmentMetadata.setSize(fileSegment.getCommitPosition());
|
|||
|
|
+- segmentMetadata.setEndTimestamp(fileSegment.getMaxTimestamp());
|
|||
|
|
+-
|
|||
|
|
+- FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment(
|
|||
|
|
+- this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset());
|
|||
|
|
+-
|
|||
|
|
+- if (!Objects.equals(metadata, segmentMetadata)) {
|
|||
|
|
+- this.tieredMetadataStore.updateFileSegment(segmentMetadata);
|
|||
|
|
+- logger.debug("TieredFlatFile#UpdateSegmentMetadata, filePath: {}, content: {}",
|
|||
|
|
+- segmentMetadata.getPath(), JSON.toJSONString(segmentMetadata));
|
|||
|
|
+- }
|
|||
|
|
++ this.tieredMetadataStore.updateFileSegment(metadata);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void checkAndFixFileSize() {
|
|||
|
|
+@@ -598,6 +578,9 @@ public class TieredFlatFile {
|
|||
|
|
+ logger.error("TieredFlatFile#destroy: mark file segment: {} is deleted failed", fileSegment.getPath(), e);
|
|||
|
|
+ }
|
|||
|
|
+ fileSegment.destroyFile();
|
|||
|
|
++ if (!fileSegment.exists()) {
|
|||
|
|
++ tieredMetadataStore.deleteFileSegment(filePath, fileType, fileSegment.getBaseOffset());
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+ fileSegmentList.clear();
|
|||
|
|
+ needCommitFileSegmentList.clear();
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
|
|||
|
|
+index 7c744af3b..087ea8c9c 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
|
|||
|
|
+@@ -136,15 +136,13 @@ public class TieredFlatFileManager {
|
|||
|
|
+ TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime());
|
|||
|
|
+ for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) {
|
|||
|
|
+ TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> {
|
|||
|
|
+- flatFile.getCompositeFlatFileLock().lock();
|
|||
|
|
+ try {
|
|||
|
|
++ flatFile.getCompositeFlatFileLock().lock();
|
|||
|
|
+ flatFile.cleanExpiredFile(expiredTimeStamp);
|
|||
|
|
+ flatFile.destroyExpiredFile();
|
|||
|
|
+- if (flatFile.getConsumeQueueBaseOffset() == -1) {
|
|||
|
|
+- logger.info("Clean flatFile because file not initialized, topic={}, queueId={}",
|
|||
|
|
+- flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId());
|
|||
|
|
+- destroyCompositeFile(flatFile.getMessageQueue());
|
|||
|
|
+- }
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ logger.error("Do Clean expired file error, topic={}, queueId={}",
|
|||
|
|
++ flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), t);
|
|||
|
|
+ } finally {
|
|||
|
|
+ flatFile.getCompositeFlatFileLock().unlock();
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 4a8e0d5b851d1f9573cda79b7d2e42ee498809da Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: guyinyou <36399867+guyinyou@users.noreply.github.com>
|
|||
|
|
+Date: Wed, 13 Sep 2023 16:08:03 +0800
|
|||
|
|
+Subject: [PATCH 3/5] [ISSUE #7351] Allow mqadmin to operate slave nodes
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: guyinyou <guyinyou.gyy@alibaba-inc.com>
|
|||
|
|
+---
|
|||
|
|
+ .../processor/AdminBrokerProcessor.java | 12 --
|
|||
|
|
+ .../processor/AdminBrokerProcessorTest.java | 106 ------------------
|
|||
|
|
+ 2 files changed, 118 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
+index 8fbcd3c94..9e48431be 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
+@@ -406,9 +406,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
+ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx,
|
|||
|
|
+ RemotingCommand request) throws RemotingCommandException {
|
|||
|
|
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+- if (validateSlave(response)) {
|
|||
|
|
+- return response;
|
|||
|
|
+- }
|
|||
|
|
+ final CreateTopicRequestHeader requestHeader =
|
|||
|
|
+ (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class);
|
|||
|
|
+
|
|||
|
|
+@@ -519,9 +516,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
+ private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx,
|
|||
|
|
+ RemotingCommand request) throws RemotingCommandException {
|
|||
|
|
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+- if (validateSlave(response)) {
|
|||
|
|
+- return response;
|
|||
|
|
+- }
|
|||
|
|
+ DeleteTopicRequestHeader requestHeader =
|
|||
|
|
+ (DeleteTopicRequestHeader) request.decodeCommandCustomHeader(DeleteTopicRequestHeader.class);
|
|||
|
|
+
|
|||
|
|
+@@ -1413,9 +1407,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
+ private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request)
|
|||
|
|
+ throws RemotingCommandException {
|
|||
|
|
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+- if (validateSlave(response)) {
|
|||
|
|
+- return response;
|
|||
|
|
+- }
|
|||
|
|
+
|
|||
|
|
+ LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}",
|
|||
|
|
+ RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
|
|||
|
|
+@@ -1480,9 +1471,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
+ private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx,
|
|||
|
|
+ RemotingCommand request) throws RemotingCommandException {
|
|||
|
|
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+- if (validateSlave(response)) {
|
|||
|
|
+- return response;
|
|||
|
|
+- }
|
|||
|
|
+ DeleteSubscriptionGroupRequestHeader requestHeader =
|
|||
|
|
+ (DeleteSubscriptionGroupRequestHeader) request.decodeCommandCustomHeader(DeleteSubscriptionGroupRequestHeader.class);
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
|
|||
|
|
+index 9d17011b6..ec252cece 100644
|
|||
|
|
+--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
|
|||
|
|
++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
|
|||
|
|
+@@ -76,7 +76,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfi
|
|||
|
|
+ import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
+ import org.apache.rocketmq.store.MessageStore;
|
|||
|
|
+ import org.apache.rocketmq.store.SelectMappedBufferResult;
|
|||
|
|
+-import org.apache.rocketmq.store.config.BrokerRole;
|
|||
|
|
+ import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
+ import org.apache.rocketmq.store.logfile.DefaultMappedFile;
|
|||
|
|
+ import org.apache.rocketmq.store.stats.BrokerStats;
|
|||
|
|
+@@ -250,32 +249,6 @@ public class AdminBrokerProcessorTest {
|
|||
|
|
+ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- @Test
|
|||
|
|
+- public void testUpdateAndCreateTopicOnSlaveInRocksdb() throws Exception {
|
|||
|
|
+- if (notToBeExecuted()) {
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- initRocksdbTopicManager();
|
|||
|
|
+- testUpdateAndCreateTopicOnSlave();
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- @Test
|
|||
|
|
+- public void testUpdateAndCreateTopicOnSlave() throws Exception {
|
|||
|
|
+- // setup
|
|||
|
|
+- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class);
|
|||
|
|
+- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE);
|
|||
|
|
+- defaultMessageStore = mock(DefaultMessageStore.class);
|
|||
|
|
+- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
|
|||
|
|
+-
|
|||
|
|
+- // test on slave
|
|||
|
|
+- String topic = "TEST_CREATE_TOPIC";
|
|||
|
|
+- RemotingCommand request = buildCreateTopicRequest(topic);
|
|||
|
|
+- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
|
|||
|
|
+- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
|
|||
|
|
+- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " +
|
|||
|
|
+- "please execute it from master broker.");
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testDeleteTopicInRocksdb() throws Exception {
|
|||
|
|
+ if (notToBeExecuted()) {
|
|||
|
|
+@@ -301,31 +274,6 @@ public class AdminBrokerProcessorTest {
|
|||
|
|
+ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- @Test
|
|||
|
|
+- public void testDeleteTopicOnSlaveInRocksdb() throws Exception {
|
|||
|
|
+- if (notToBeExecuted()) {
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- initRocksdbTopicManager();
|
|||
|
|
+- testDeleteTopicOnSlave();
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- @Test
|
|||
|
|
+- public void testDeleteTopicOnSlave() throws Exception {
|
|||
|
|
+- // setup
|
|||
|
|
+- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class);
|
|||
|
|
+- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE);
|
|||
|
|
+- defaultMessageStore = mock(DefaultMessageStore.class);
|
|||
|
|
+- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
|
|||
|
|
+-
|
|||
|
|
+- String topic = "TEST_DELETE_TOPIC";
|
|||
|
|
+- RemotingCommand request = buildDeleteTopicRequest(topic);
|
|||
|
|
+- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
|
|||
|
|
+- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
|
|||
|
|
+- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " +
|
|||
|
|
+- "please execute it from master broker.");
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testDeleteWithPopRetryTopic() throws Exception {
|
|||
|
|
+ String topic = "topicA";
|
|||
|
|
+@@ -538,36 +486,6 @@ public class AdminBrokerProcessorTest {
|
|||
|
|
+ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- @Test
|
|||
|
|
+- public void testUpdateAndCreateSubscriptionGroupOnSlaveInRocksdb() throws Exception {
|
|||
|
|
+- initRocksdbSubscriptionManager();
|
|||
|
|
+- testUpdateAndCreateSubscriptionGroupOnSlave();
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- @Test
|
|||
|
|
+- public void testUpdateAndCreateSubscriptionGroupOnSlave() throws RemotingCommandException {
|
|||
|
|
+- // Setup
|
|||
|
|
+- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class);
|
|||
|
|
+- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE);
|
|||
|
|
+- defaultMessageStore = mock(DefaultMessageStore.class);
|
|||
|
|
+- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
|
|||
|
|
+-
|
|||
|
|
+- // Test
|
|||
|
|
+- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null);
|
|||
|
|
+- SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
|
|||
|
|
+- subscriptionGroupConfig.setBrokerId(1);
|
|||
|
|
+- subscriptionGroupConfig.setGroupName("groupId");
|
|||
|
|
+- subscriptionGroupConfig.setConsumeEnable(Boolean.TRUE);
|
|||
|
|
+- subscriptionGroupConfig.setConsumeBroadcastEnable(Boolean.TRUE);
|
|||
|
|
+- subscriptionGroupConfig.setRetryMaxTimes(111);
|
|||
|
|
+- subscriptionGroupConfig.setConsumeFromMinEnable(Boolean.TRUE);
|
|||
|
|
+- request.setBody(JSON.toJSON(subscriptionGroupConfig).toString().getBytes());
|
|||
|
|
+- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
|
|||
|
|
+- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
|
|||
|
|
+- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " +
|
|||
|
|
+- "please execute it from master broker.");
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetAllSubscriptionGroupInRocksdb() throws Exception {
|
|||
|
|
+ initRocksdbSubscriptionManager();
|
|||
|
|
+@@ -596,30 +514,6 @@ public class AdminBrokerProcessorTest {
|
|||
|
|
+ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- @Test
|
|||
|
|
+- public void testDeleteSubscriptionGroupOnSlaveInRocksdb() throws Exception {
|
|||
|
|
+- initRocksdbSubscriptionManager();
|
|||
|
|
+- testDeleteSubscriptionGroupOnSlave();
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- @Test
|
|||
|
|
+- public void testDeleteSubscriptionGroupOnSlave() throws RemotingCommandException {
|
|||
|
|
+- // Setup
|
|||
|
|
+- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class);
|
|||
|
|
+- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE);
|
|||
|
|
+- defaultMessageStore = mock(DefaultMessageStore.class);
|
|||
|
|
+- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
|
|||
|
|
+-
|
|||
|
|
+- // Test
|
|||
|
|
+- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, null);
|
|||
|
|
+- request.addExtField("groupName", "GID-Group-Name");
|
|||
|
|
+- request.addExtField("removeOffset", "true");
|
|||
|
|
+- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
|
|||
|
|
+- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
|
|||
|
|
+- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " +
|
|||
|
|
+- "please execute it from master broker.");
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetTopicStatsInfo() throws RemotingCommandException {
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, null);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 831fcc76cd7cd362bb6c136c287c624bb7eaf40a Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lizhimins <707364882@qq.com>
|
|||
|
|
+Date: Tue, 19 Sep 2023 10:04:04 +0800
|
|||
|
|
+Subject: [PATCH 4/5] [ISSUE #7363] Fix get message from tiered storage return
|
|||
|
|
+ incorrect next pull offset (#7365)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../tieredstore/TieredMessageFetcher.java | 2 +-
|
|||
|
|
+ .../tieredstore/TieredMessageStore.java | 29 ++++++++++---------
|
|||
|
|
+ .../tieredstore/TieredMessageStoreTest.java | 5 ++--
|
|||
|
|
+ 3 files changed, 20 insertions(+), 16 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java
|
|||
|
|
+index 766ff64f6..c948fa3fa 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java
|
|||
|
|
+@@ -319,7 +319,7 @@ public class TieredMessageFetcher implements MessageStoreFetcher {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ // if cache is miss, immediately pull messages
|
|||
|
|
+- LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " +
|
|||
|
|
++ LOGGER.info("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " +
|
|||
|
|
+ "topic: {}, queue: {}, queue offset: {}, max message num: {}",
|
|||
|
|
+ mq.getTopic(), mq.getQueueId(), queueOffset, maxCount);
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+index 9fb1b2f01..d7d13d61e 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+@@ -147,6 +147,11 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ public CompletableFuture<GetMessageResult> getMessageAsync(String group, String topic,
|
|||
|
|
+ int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) {
|
|||
|
|
+
|
|||
|
|
++ // For system topic, force reading from local store
|
|||
|
|
++ if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) {
|
|||
|
|
++ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) {
|
|||
|
|
+ logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset);
|
|||
|
|
+ } else {
|
|||
|
|
+@@ -158,6 +163,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ return fetcher
|
|||
|
|
+ .getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter)
|
|||
|
|
+ .thenApply(result -> {
|
|||
|
|
++
|
|||
|
|
+ Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder()
|
|||
|
|
+ .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_GET_MESSAGE)
|
|||
|
|
+ .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)
|
|||
|
|
+@@ -166,8 +172,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);
|
|||
|
|
+
|
|||
|
|
+ if (result.getStatus() == GetMessageStatus.OFFSET_FOUND_NULL ||
|
|||
|
|
+- result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_ONE ||
|
|||
|
|
+- result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_BADLY) {
|
|||
|
|
++ result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) {
|
|||
|
|
+
|
|||
|
|
+ if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) {
|
|||
|
|
+ TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes);
|
|||
|
|
+@@ -178,14 +183,8 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- // Fetch system topic data from the broker when using the force level.
|
|||
|
|
+- if (result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) {
|
|||
|
|
+- if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) {
|
|||
|
|
+- return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+ if (result.getStatus() != GetMessageStatus.FOUND &&
|
|||
|
|
++ result.getStatus() != GetMessageStatus.NO_MATCHED_LOGIC_QUEUE &&
|
|||
|
|
+ result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_ONE &&
|
|||
|
|
+ result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_BADLY) {
|
|||
|
|
+ logger.warn("GetMessageAsync not found and message is not in next store, result: {}, " +
|
|||
|
|
+@@ -206,10 +205,14 @@ public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
+ if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) {
|
|||
|
|
+ result.setMinOffset(minOffsetInQueue);
|
|||
|
|
+ }
|
|||
|
|
+- long maxOffsetInQueue = next.getMaxOffsetInQueue(topic, queueId);
|
|||
|
|
+- if (maxOffsetInQueue >= 0 && maxOffsetInQueue > result.getMaxOffset()) {
|
|||
|
|
+- result.setMaxOffset(maxOffsetInQueue);
|
|||
|
|
+- }
|
|||
|
|
++
|
|||
|
|
++ // In general, the local cq offset is slightly greater than the commit offset in read message,
|
|||
|
|
++ // so there is no need to update the maximum offset to the local cq offset here,
|
|||
|
|
++ // otherwise it will cause repeated consumption after next begin offset over commit offset.
|
|||
|
|
++
|
|||
|
|
++ logger.trace("GetMessageAsync result, group: {}, topic: {}, queueId: {}, offset: {}, count:{}, {}",
|
|||
|
|
++ group, topic, queueId, offset, maxMsgNums, result);
|
|||
|
|
++
|
|||
|
|
+ return result;
|
|||
|
|
+ }).exceptionally(e -> {
|
|||
|
|
+ logger.error("GetMessageAsync from tiered store failed", e);
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java
|
|||
|
|
+index 2451199c2..07af1fc8b 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java
|
|||
|
|
+@@ -168,7 +168,7 @@ public class TieredMessageStoreTest {
|
|||
|
|
+ GetMessageResult result1 = new GetMessageResult();
|
|||
|
|
+ result1.setStatus(GetMessageStatus.FOUND);
|
|||
|
|
+ GetMessageResult result2 = new GetMessageResult();
|
|||
|
|
+- result2.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING);
|
|||
|
|
++ result2.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
|
|||
|
|
+
|
|||
|
|
+ when(fetcher.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(result1));
|
|||
|
|
+ when(nextStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(result2);
|
|||
|
|
+@@ -188,7 +188,8 @@ public class TieredMessageStoreTest {
|
|||
|
|
+ properties.setProperty("tieredStorageLevel", "3");
|
|||
|
|
+ configuration.update(properties);
|
|||
|
|
+ when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
|
|||
|
|
+- Assert.assertSame(result2, store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null));
|
|||
|
|
++ Assert.assertEquals(result2.getStatus(),
|
|||
|
|
++ store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null).getStatus());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From f05a8da760dfade411ad56ef874f477988479cf9 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: rongtong <jinrongtong5@163.com>
|
|||
|
|
+Date: Wed, 20 Sep 2023 15:06:21 +0800
|
|||
|
|
+Subject: [PATCH 5/5] Print admin queue watermark in log (#7372)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 1 +
|
|||
|
|
+ 1 file changed, 1 insertion(+)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+index 13a3feb4e..53e2e1b62 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+@@ -1182,6 +1182,7 @@ public class BrokerController {
|
|||
|
|
+ LOG_WATER_MARK.info("[WATERMARK] ClientManager Queue Size: {} SlowTimeMills: {}", this.clientManagerThreadPoolQueue.size(), this.headSlowTimeMills(this.clientManagerThreadPoolQueue));
|
|||
|
|
+ LOG_WATER_MARK.info("[WATERMARK] Heartbeat Queue Size: {} SlowTimeMills: {}", this.heartbeatThreadPoolQueue.size(), this.headSlowTimeMills(this.heartbeatThreadPoolQueue));
|
|||
|
|
+ LOG_WATER_MARK.info("[WATERMARK] Ack Queue Size: {} SlowTimeMills: {}", this.ackThreadPoolQueue.size(), headSlowTimeMills(this.ackThreadPoolQueue));
|
|||
|
|
++ LOG_WATER_MARK.info("[WATERMARK] Admin Queue Size: {} SlowTimeMills: {}", this.adminBrokerThreadPoolQueue.size(), headSlowTimeMills(this.adminBrokerThreadPoolQueue));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public MessageStore getMessageStore() {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch019-backport-some-bugfix.patch b/patch019-backport-some-bugfix.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..d85f01c98
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch019-backport-some-bugfix.patch
|
|||
|
|
@@ -0,0 +1,1499 @@
|
|||
|
|
+From 42fcd278ca84f6988d48a7d11271fc81b921d59a Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lk <xdkxlk@outlook.com>
|
|||
|
|
+Date: Wed, 20 Sep 2023 15:41:23 +0800
|
|||
|
|
+Subject: [PATCH 01/12] [ISSUE #7374] Prepare to release Apache RocketMQ 5.1.4
|
|||
|
|
+ (#7375)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ common/src/main/java/org/apache/rocketmq/common/MQVersion.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
|
|||
|
|
+index bfd07a895..4f1990ff8 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
|
|||
|
|
+@@ -18,7 +18,7 @@ package org.apache.rocketmq.common;
|
|||
|
|
+
|
|||
|
|
+ public class MQVersion {
|
|||
|
|
+
|
|||
|
|
+- public static final int CURRENT_VERSION = Version.V5_1_3.ordinal();
|
|||
|
|
++ public static final int CURRENT_VERSION = Version.V5_1_4.ordinal();
|
|||
|
|
+
|
|||
|
|
+ public static String getVersionDesc(int value) {
|
|||
|
|
+ int length = Version.values().length;
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From b8610d87bb55de1f4413460c05da529dab60c1c1 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Jixiang Jin <lollipop@apache.org>
|
|||
|
|
+Date: Thu, 21 Sep 2023 16:21:44 +0800
|
|||
|
|
+Subject: [PATCH 02/12] Replace loggingMetricExporter with
|
|||
|
|
+ OtlpJsonLoggingMetricExporter. (#7373)
|
|||
|
|
+
|
|||
|
|
+* Replace loggingMetricExporter with OtlpJsonLoggingMetricExporter.
|
|||
|
|
+
|
|||
|
|
+* Fix bazel workspace
|
|||
|
|
+
|
|||
|
|
+* Move OtlpJsonLoggingMetricExporter to proxy and controller.
|
|||
|
|
+
|
|||
|
|
+* Fix Bazel imports.
|
|||
|
|
+---
|
|||
|
|
+ WORKSPACE | 1 +
|
|||
|
|
+ broker/BUILD.bazel | 1 +
|
|||
|
|
+ .../rocketmq/broker/metrics/BrokerMetricsManager.java | 9 +++++----
|
|||
|
|
+ broker/src/main/resources/rmq.broker.logback.xml | 5 +++++
|
|||
|
|
+ common/BUILD.bazel | 1 +
|
|||
|
|
+ common/pom.xml | 4 ++++
|
|||
|
|
+ controller/BUILD.bazel | 1 +
|
|||
|
|
+ .../controller/metrics/ControllerMetricsManager.java | 9 +++++----
|
|||
|
|
+ pom.xml | 5 +++++
|
|||
|
|
+ proxy/BUILD.bazel | 1 +
|
|||
|
|
+ .../rocketmq/proxy/metrics/ProxyMetricsManager.java | 11 ++++++-----
|
|||
|
|
+ proxy/src/main/resources/rmq.proxy.logback.xml | 5 +++++
|
|||
|
|
+ tieredstore/BUILD.bazel | 1 +
|
|||
|
|
+ 13 files changed, 41 insertions(+), 13 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/WORKSPACE b/WORKSPACE
|
|||
|
|
+index 3126f2d1d..8640485ba 100644
|
|||
|
|
+--- a/WORKSPACE
|
|||
|
|
++++ b/WORKSPACE
|
|||
|
|
+@@ -92,6 +92,7 @@ maven_install(
|
|||
|
|
+ "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha",
|
|||
|
|
+ "io.opentelemetry:opentelemetry-exporter-logging:1.29.0",
|
|||
|
|
+ "io.opentelemetry:opentelemetry-sdk:1.29.0",
|
|||
|
|
++ "io.opentelemetry:opentelemetry-exporter-logging-otlp:1.29.0",
|
|||
|
|
+ "com.squareup.okio:okio-jvm:3.0.0",
|
|||
|
|
+ "io.opentelemetry:opentelemetry-api:1.29.0",
|
|||
|
|
+ "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0",
|
|||
|
|
+diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel
|
|||
|
|
+index 6adcdc7b9..64cb1b341 100644
|
|||
|
|
+--- a/broker/BUILD.bazel
|
|||
|
|
++++ b/broker/BUILD.bazel
|
|||
|
|
+@@ -44,6 +44,7 @@ java_library(
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_exporter_otlp",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_exporter_logging",
|
|||
|
|
++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_common",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics",
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
|
|||
|
|
+index 6af5afc14..39af18b9f 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
|
|||
|
|
+@@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongCounter;
|
|||
|
|
+ import io.opentelemetry.api.metrics.LongHistogram;
|
|||
|
|
+ import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+ import io.opentelemetry.api.metrics.ObservableLongGauge;
|
|||
|
|
+-import io.opentelemetry.exporter.logging.LoggingMetricExporter;
|
|||
|
|
++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter;
|
|||
|
|
+ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter;
|
|||
|
|
+ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder;
|
|||
|
|
+ import io.opentelemetry.exporter.prometheus.PrometheusHttpServer;
|
|||
|
|
+@@ -36,6 +36,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.export.MetricExporter;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil;
|
|||
|
|
+ import io.opentelemetry.sdk.resources.Resource;
|
|||
|
|
+@@ -113,7 +114,7 @@ public class BrokerMetricsManager {
|
|||
|
|
+ private OtlpGrpcMetricExporter metricExporter;
|
|||
|
|
+ private PeriodicMetricReader periodicMetricReader;
|
|||
|
|
+ private PrometheusHttpServer prometheusHttpServer;
|
|||
|
|
+- private LoggingMetricExporter loggingMetricExporter;
|
|||
|
|
++ private MetricExporter loggingMetricExporter;
|
|||
|
|
+ private Meter brokerMeter;
|
|||
|
|
+
|
|||
|
|
+ public static Supplier<AttributesBuilder> attributesBuilderSupplier = Attributes::builder;
|
|||
|
|
+@@ -327,8 +328,8 @@ public class BrokerMetricsManager {
|
|||
|
|
+ if (metricsExporterType == MetricsExporterType.LOG) {
|
|||
|
|
+ SLF4JBridgeHandler.removeHandlersForRootLogger();
|
|||
|
|
+ SLF4JBridgeHandler.install();
|
|||
|
|
+- loggingMetricExporter = LoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE);
|
|||
|
|
+- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST);
|
|||
|
|
++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE);
|
|||
|
|
++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST);
|
|||
|
|
+ periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter)
|
|||
|
|
+ .setInterval(brokerConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS)
|
|||
|
|
+ .build();
|
|||
|
|
+diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml
|
|||
|
|
+index 3c51e59d4..32dc29736 100644
|
|||
|
|
+--- a/broker/src/main/resources/rmq.broker.logback.xml
|
|||
|
|
++++ b/broker/src/main/resources/rmq.broker.logback.xml
|
|||
|
|
+@@ -672,6 +672,11 @@
|
|||
|
|
+ <appender-ref ref="RocketmqTrafficSiftingAppender"/>
|
|||
|
|
+ </logger>
|
|||
|
|
+
|
|||
|
|
++ <!-- Use json formatter to log metrics -->
|
|||
|
|
++ <logger name="io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter" additivity="false" level="INFO">
|
|||
|
|
++ <appender-ref ref="RocketmqBrokerMetricsSiftingAppender"/>
|
|||
|
|
++ </logger>
|
|||
|
|
++
|
|||
|
|
+ <logger name="io.opentelemetry.exporter.logging.LoggingMetricExporter" additivity="false" level="INFO">
|
|||
|
|
+ <appender-ref ref="RocketmqBrokerMetricsSiftingAppender"/>
|
|||
|
|
+ </logger>
|
|||
|
|
+diff --git a/common/BUILD.bazel b/common/BUILD.bazel
|
|||
|
|
+index a95a19ccd..e6701d0fc 100644
|
|||
|
|
+--- a/common/BUILD.bazel
|
|||
|
|
++++ b/common/BUILD.bazel
|
|||
|
|
+@@ -35,6 +35,7 @@ java_library(
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_common",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics",
|
|||
|
|
++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp",
|
|||
|
|
+ "@maven//:org_apache_commons_commons_lang3",
|
|||
|
|
+ "@maven//:org_lz4_lz4_java",
|
|||
|
|
+ "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
|
|||
|
|
+diff --git a/common/pom.xml b/common/pom.xml
|
|||
|
|
+index 31eb0f087..accc7f0a8 100644
|
|||
|
|
+--- a/common/pom.xml
|
|||
|
|
++++ b/common/pom.xml
|
|||
|
|
+@@ -80,6 +80,10 @@
|
|||
|
|
+ <groupId>io.opentelemetry</groupId>
|
|||
|
|
+ <artifactId>opentelemetry-sdk</artifactId>
|
|||
|
|
+ </dependency>
|
|||
|
|
++ <dependency>
|
|||
|
|
++ <groupId>io.opentelemetry</groupId>
|
|||
|
|
++ <artifactId>opentelemetry-exporter-logging-otlp</artifactId>
|
|||
|
|
++ </dependency>
|
|||
|
|
+ <dependency>
|
|||
|
|
+ <groupId>io.grpc</groupId>
|
|||
|
|
+ <artifactId>grpc-stub</artifactId>
|
|||
|
|
+diff --git a/controller/BUILD.bazel b/controller/BUILD.bazel
|
|||
|
|
+index 843d9dc77..b2b743eb2 100644
|
|||
|
|
+--- a/controller/BUILD.bazel
|
|||
|
|
++++ b/controller/BUILD.bazel
|
|||
|
|
+@@ -49,6 +49,7 @@ java_library(
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_common",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_exporter_logging",
|
|||
|
|
++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp",
|
|||
|
|
+ "@maven//:org_slf4j_jul_to_slf4j",
|
|||
|
|
+ ],
|
|||
|
|
+ )
|
|||
|
|
+diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java
|
|||
|
|
+index 650740bcc..be9e77eea 100644
|
|||
|
|
+--- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java
|
|||
|
|
++++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java
|
|||
|
|
+@@ -26,7 +26,7 @@ import io.opentelemetry.api.metrics.LongHistogram;
|
|||
|
|
+ import io.opentelemetry.api.metrics.LongUpDownCounter;
|
|||
|
|
+ import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+ import io.opentelemetry.api.metrics.ObservableLongGauge;
|
|||
|
|
+-import io.opentelemetry.exporter.logging.LoggingMetricExporter;
|
|||
|
|
++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter;
|
|||
|
|
+ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter;
|
|||
|
|
+ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder;
|
|||
|
|
+ import io.opentelemetry.exporter.prometheus.PrometheusHttpServer;
|
|||
|
|
+@@ -38,6 +38,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProvider;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.View;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.export.MetricExporter;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
|
|||
|
|
+ import io.opentelemetry.sdk.resources.Resource;
|
|||
|
|
+ import java.io.File;
|
|||
|
|
+@@ -121,7 +122,7 @@ public class ControllerMetricsManager {
|
|||
|
|
+
|
|||
|
|
+ private PrometheusHttpServer prometheusHttpServer;
|
|||
|
|
+
|
|||
|
|
+- private LoggingMetricExporter loggingMetricExporter;
|
|||
|
|
++ private MetricExporter loggingMetricExporter;
|
|||
|
|
+
|
|||
|
|
+ public static ControllerMetricsManager getInstance(ControllerManager controllerManager) {
|
|||
|
|
+ if (instance == null) {
|
|||
|
|
+@@ -364,8 +365,8 @@ public class ControllerMetricsManager {
|
|||
|
|
+ if (type == MetricsExporterType.LOG) {
|
|||
|
|
+ SLF4JBridgeHandler.removeHandlersForRootLogger();
|
|||
|
|
+ SLF4JBridgeHandler.install();
|
|||
|
|
+- loggingMetricExporter = LoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE);
|
|||
|
|
+- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST);
|
|||
|
|
++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE);
|
|||
|
|
++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST);
|
|||
|
|
+ periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter)
|
|||
|
|
+ .setInterval(config.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS)
|
|||
|
|
+ .build();
|
|||
|
|
+diff --git a/pom.xml b/pom.xml
|
|||
|
|
+index 9f0b3eb96..4b382c6da 100644
|
|||
|
|
+--- a/pom.xml
|
|||
|
|
++++ b/pom.xml
|
|||
|
|
+@@ -974,6 +974,11 @@
|
|||
|
|
+ <artifactId>opentelemetry-sdk</artifactId>
|
|||
|
|
+ <version>${opentelemetry.version}</version>
|
|||
|
|
+ </dependency>
|
|||
|
|
++ <dependency>
|
|||
|
|
++ <groupId>io.opentelemetry</groupId>
|
|||
|
|
++ <artifactId>opentelemetry-exporter-logging-otlp</artifactId>
|
|||
|
|
++ <version>${opentelemetry.version}</version>
|
|||
|
|
++ </dependency>
|
|||
|
|
+ <dependency>
|
|||
|
|
+ <groupId>org.slf4j</groupId>
|
|||
|
|
+ <artifactId>jul-to-slf4j</artifactId>
|
|||
|
|
+diff --git a/proxy/BUILD.bazel b/proxy/BUILD.bazel
|
|||
|
|
+index b4f3c16e2..cb7af9254 100644
|
|||
|
|
+--- a/proxy/BUILD.bazel
|
|||
|
|
++++ b/proxy/BUILD.bazel
|
|||
|
|
+@@ -52,6 +52,7 @@ java_library(
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_exporter_otlp",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_exporter_logging",
|
|||
|
|
++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_common",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics",
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java
|
|||
|
|
+index f5050858f..2b8dac5d8 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java
|
|||
|
|
+@@ -21,15 +21,16 @@ import io.opentelemetry.api.common.Attributes;
|
|||
|
|
+ import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
+ import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+ import io.opentelemetry.api.metrics.ObservableLongGauge;
|
|||
|
|
++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter;
|
|||
|
|
+ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter;
|
|||
|
|
+ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder;
|
|||
|
|
+ import io.opentelemetry.exporter.prometheus.PrometheusHttpServer;
|
|||
|
|
+-import io.opentelemetry.exporter.logging.LoggingMetricExporter;
|
|||
|
|
+ import io.opentelemetry.sdk.OpenTelemetrySdk;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.InstrumentType;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.SdkMeterProvider;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
|
|||
|
|
++import io.opentelemetry.sdk.metrics.export.MetricExporter;
|
|||
|
|
+ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
|
|||
|
|
+ import io.opentelemetry.sdk.resources.Resource;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+@@ -42,9 +43,9 @@ import org.apache.rocketmq.broker.metrics.BrokerMetricsManager;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.metrics.MetricsExporterType;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.StartAndShutdown;
|
|||
|
|
+-import org.apache.rocketmq.proxy.config.ProxyConfig;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
++import org.apache.rocketmq.proxy.config.ProxyConfig;
|
|||
|
|
+ import org.slf4j.bridge.SLF4JBridgeHandler;
|
|||
|
|
+
|
|||
|
|
+ import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.AGGREGATION_DELTA;
|
|||
|
|
+@@ -67,7 +68,7 @@ public class ProxyMetricsManager implements StartAndShutdown {
|
|||
|
|
+ private OtlpGrpcMetricExporter metricExporter;
|
|||
|
|
+ private PeriodicMetricReader periodicMetricReader;
|
|||
|
|
+ private PrometheusHttpServer prometheusHttpServer;
|
|||
|
|
+- private LoggingMetricExporter loggingMetricExporter;
|
|||
|
|
++ private MetricExporter loggingMetricExporter;
|
|||
|
|
+
|
|||
|
|
+ public static ObservableLongGauge proxyUp = null;
|
|||
|
|
+
|
|||
|
|
+@@ -221,8 +222,8 @@ public class ProxyMetricsManager implements StartAndShutdown {
|
|||
|
|
+ if (metricsExporterType == MetricsExporterType.LOG) {
|
|||
|
|
+ SLF4JBridgeHandler.removeHandlersForRootLogger();
|
|||
|
|
+ SLF4JBridgeHandler.install();
|
|||
|
|
+- loggingMetricExporter = LoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE);
|
|||
|
|
+- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST);
|
|||
|
|
++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE);
|
|||
|
|
++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST);
|
|||
|
|
+ periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter)
|
|||
|
|
+ .setInterval(proxyConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS)
|
|||
|
|
+ .build();
|
|||
|
|
+diff --git a/proxy/src/main/resources/rmq.proxy.logback.xml b/proxy/src/main/resources/rmq.proxy.logback.xml
|
|||
|
|
+index d38827f92..f968a45e6 100644
|
|||
|
|
+--- a/proxy/src/main/resources/rmq.proxy.logback.xml
|
|||
|
|
++++ b/proxy/src/main/resources/rmq.proxy.logback.xml
|
|||
|
|
+@@ -418,6 +418,11 @@
|
|||
|
|
+ <appender-ref ref="RocketmqProxyWatermarkAppender" />
|
|||
|
|
+ </logger>
|
|||
|
|
+
|
|||
|
|
++ <!-- Use json formatter to log metrics -->
|
|||
|
|
++ <logger name="io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter" additivity="false" level="INFO">
|
|||
|
|
++ <appender-ref ref="RocketmqProxyMetricsAppender"/>
|
|||
|
|
++ </logger>
|
|||
|
|
++
|
|||
|
|
+ <logger name="io.opentelemetry.exporter.logging.LoggingMetricExporter" additivity="false" level="INFO">
|
|||
|
|
+ <appender-ref ref="RocketmqProxyMetricsAppender" />
|
|||
|
|
+ </logger>
|
|||
|
|
+diff --git a/tieredstore/BUILD.bazel b/tieredstore/BUILD.bazel
|
|||
|
|
+index 5b3885a4e..dea2c561b 100644
|
|||
|
|
+--- a/tieredstore/BUILD.bazel
|
|||
|
|
++++ b/tieredstore/BUILD.bazel
|
|||
|
|
+@@ -36,6 +36,7 @@ java_library(
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_common",
|
|||
|
|
+ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics",
|
|||
|
|
++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp",
|
|||
|
|
+ "@maven//:org_apache_commons_commons_lang3",
|
|||
|
|
+ "@maven//:org_apache_tomcat_annotations_api",
|
|||
|
|
+ "@maven//:com_alibaba_fastjson",
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 1a681bdf9b5c5ab0be446d6394c0cac8768f45d9 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lk <xdkxlk@outlook.com>
|
|||
|
|
+Date: Thu, 21 Sep 2023 19:58:29 +0800
|
|||
|
|
+Subject: [PATCH 03/12] [maven-release-plugin] prepare release
|
|||
|
|
+ rocketmq-all-5.1.4 (#7377)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ acl/pom.xml | 2 +-
|
|||
|
|
+ broker/pom.xml | 2 +-
|
|||
|
|
+ client/pom.xml | 2 +-
|
|||
|
|
+ common/pom.xml | 2 +-
|
|||
|
|
+ container/pom.xml | 2 +-
|
|||
|
|
+ controller/pom.xml | 2 +-
|
|||
|
|
+ distribution/pom.xml | 2 +-
|
|||
|
|
+ example/pom.xml | 2 +-
|
|||
|
|
+ filter/pom.xml | 2 +-
|
|||
|
|
+ namesrv/pom.xml | 2 +-
|
|||
|
|
+ openmessaging/pom.xml | 2 +-
|
|||
|
|
+ pom.xml | 4 ++--
|
|||
|
|
+ proxy/pom.xml | 2 +-
|
|||
|
|
+ remoting/pom.xml | 2 +-
|
|||
|
|
+ srvutil/pom.xml | 2 +-
|
|||
|
|
+ store/pom.xml | 2 +-
|
|||
|
|
+ test/pom.xml | 2 +-
|
|||
|
|
+ tieredstore/pom.xml | 2 +-
|
|||
|
|
+ tools/pom.xml | 2 +-
|
|||
|
|
+ 19 files changed, 20 insertions(+), 20 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/acl/pom.xml b/acl/pom.xml
|
|||
|
|
+index 989c0cf77..9f6838b00 100644
|
|||
|
|
+--- a/acl/pom.xml
|
|||
|
|
++++ b/acl/pom.xml
|
|||
|
|
+@@ -13,7 +13,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+ <artifactId>rocketmq-acl</artifactId>
|
|||
|
|
+ <name>rocketmq-acl ${project.version}</name>
|
|||
|
|
+diff --git a/broker/pom.xml b/broker/pom.xml
|
|||
|
|
+index 16e026276..d483e67ba 100644
|
|||
|
|
+--- a/broker/pom.xml
|
|||
|
|
++++ b/broker/pom.xml
|
|||
|
|
+@@ -13,7 +13,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/client/pom.xml b/client/pom.xml
|
|||
|
|
+index c59a43889..4febedc6d 100644
|
|||
|
|
+--- a/client/pom.xml
|
|||
|
|
++++ b/client/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/common/pom.xml b/common/pom.xml
|
|||
|
|
+index accc7f0a8..b70873dfa 100644
|
|||
|
|
+--- a/common/pom.xml
|
|||
|
|
++++ b/common/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/container/pom.xml b/container/pom.xml
|
|||
|
|
+index c8499f127..e6c1f4b4d 100644
|
|||
|
|
+--- a/container/pom.xml
|
|||
|
|
++++ b/container/pom.xml
|
|||
|
|
+@@ -18,7 +18,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/controller/pom.xml b/controller/pom.xml
|
|||
|
|
+index 3346c7c82..46a3834c6 100644
|
|||
|
|
+--- a/controller/pom.xml
|
|||
|
|
++++ b/controller/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+ <packaging>jar</packaging>
|
|||
|
|
+diff --git a/distribution/pom.xml b/distribution/pom.xml
|
|||
|
|
+index dbde2d9d4..346c4de35 100644
|
|||
|
|
+--- a/distribution/pom.xml
|
|||
|
|
++++ b/distribution/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+ <artifactId>rocketmq-distribution</artifactId>
|
|||
|
|
+ <name>rocketmq-distribution ${project.version}</name>
|
|||
|
|
+diff --git a/example/pom.xml b/example/pom.xml
|
|||
|
|
+index 862fc3169..9e7db43f8 100644
|
|||
|
|
+--- a/example/pom.xml
|
|||
|
|
++++ b/example/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/filter/pom.xml b/filter/pom.xml
|
|||
|
|
+index 3fe51ceae..84189066d 100644
|
|||
|
|
+--- a/filter/pom.xml
|
|||
|
|
++++ b/filter/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/namesrv/pom.xml b/namesrv/pom.xml
|
|||
|
|
+index 684b2683c..7c218078a 100644
|
|||
|
|
+--- a/namesrv/pom.xml
|
|||
|
|
++++ b/namesrv/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml
|
|||
|
|
+index aaa4c896c..fd499e3de 100644
|
|||
|
|
+--- a/openmessaging/pom.xml
|
|||
|
|
++++ b/openmessaging/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/pom.xml b/pom.xml
|
|||
|
|
+index 4b382c6da..0e1d04f15 100644
|
|||
|
|
+--- a/pom.xml
|
|||
|
|
++++ b/pom.xml
|
|||
|
|
+@@ -28,7 +28,7 @@
|
|||
|
|
+ <inceptionYear>2012</inceptionYear>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ <packaging>pom</packaging>
|
|||
|
|
+ <name>Apache RocketMQ ${project.version}</name>
|
|||
|
|
+ <url>http://rocketmq.apache.org/</url>
|
|||
|
|
+@@ -37,7 +37,7 @@
|
|||
|
|
+ <url>git@github.com:apache/rocketmq.git</url>
|
|||
|
|
+ <connection>scm:git:git@github.com:apache/rocketmq.git</connection>
|
|||
|
|
+ <developerConnection>scm:git:git@github.com:apache/rocketmq.git</developerConnection>
|
|||
|
|
+- <tag>HEAD</tag>
|
|||
|
|
++ <tag>rocketmq-all-5.1.4</tag>
|
|||
|
|
+ </scm>
|
|||
|
|
+
|
|||
|
|
+ <mailingLists>
|
|||
|
|
+diff --git a/proxy/pom.xml b/proxy/pom.xml
|
|||
|
|
+index 3fbea107a..abf242eee 100644
|
|||
|
|
+--- a/proxy/pom.xml
|
|||
|
|
++++ b/proxy/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/remoting/pom.xml b/remoting/pom.xml
|
|||
|
|
+index 8a43c5c30..fc70cb62e 100644
|
|||
|
|
+--- a/remoting/pom.xml
|
|||
|
|
++++ b/remoting/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/srvutil/pom.xml b/srvutil/pom.xml
|
|||
|
|
+index fa54ad019..d7f946cce 100644
|
|||
|
|
+--- a/srvutil/pom.xml
|
|||
|
|
++++ b/srvutil/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/store/pom.xml b/store/pom.xml
|
|||
|
|
+index 38f04009d..6d6983c5d 100644
|
|||
|
|
+--- a/store/pom.xml
|
|||
|
|
++++ b/store/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/test/pom.xml b/test/pom.xml
|
|||
|
|
+index 8f25c35c9..39090e426 100644
|
|||
|
|
+--- a/test/pom.xml
|
|||
|
|
++++ b/test/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml
|
|||
|
|
+index c476040ba..7b209751f 100644
|
|||
|
|
+--- a/tieredstore/pom.xml
|
|||
|
|
++++ b/tieredstore/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/tools/pom.xml b/tools/pom.xml
|
|||
|
|
+index 1c3b431bc..806787ec9 100644
|
|||
|
|
+--- a/tools/pom.xml
|
|||
|
|
++++ b/tools/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4-SNAPSHOT</version>
|
|||
|
|
++ <version>5.1.4</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 73b3fde83765e066541e3455cd1e6604292a9b7c Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lk <xdkxlk@outlook.com>
|
|||
|
|
+Date: Fri, 22 Sep 2023 10:08:59 +0800
|
|||
|
|
+Subject: [PATCH 04/12] [maven-release-plugin] prepare for next development
|
|||
|
|
+ iteration (#7379)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ acl/pom.xml | 2 +-
|
|||
|
|
+ broker/pom.xml | 2 +-
|
|||
|
|
+ client/pom.xml | 2 +-
|
|||
|
|
+ common/pom.xml | 2 +-
|
|||
|
|
+ container/pom.xml | 2 +-
|
|||
|
|
+ controller/pom.xml | 2 +-
|
|||
|
|
+ distribution/pom.xml | 2 +-
|
|||
|
|
+ example/pom.xml | 2 +-
|
|||
|
|
+ filter/pom.xml | 2 +-
|
|||
|
|
+ namesrv/pom.xml | 2 +-
|
|||
|
|
+ openmessaging/pom.xml | 2 +-
|
|||
|
|
+ pom.xml | 4 ++--
|
|||
|
|
+ proxy/pom.xml | 2 +-
|
|||
|
|
+ remoting/pom.xml | 2 +-
|
|||
|
|
+ srvutil/pom.xml | 2 +-
|
|||
|
|
+ store/pom.xml | 2 +-
|
|||
|
|
+ test/pom.xml | 2 +-
|
|||
|
|
+ tieredstore/pom.xml | 2 +-
|
|||
|
|
+ tools/pom.xml | 2 +-
|
|||
|
|
+ 19 files changed, 20 insertions(+), 20 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/acl/pom.xml b/acl/pom.xml
|
|||
|
|
+index 9f6838b00..8a296e5ae 100644
|
|||
|
|
+--- a/acl/pom.xml
|
|||
|
|
++++ b/acl/pom.xml
|
|||
|
|
+@@ -13,7 +13,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+ <artifactId>rocketmq-acl</artifactId>
|
|||
|
|
+ <name>rocketmq-acl ${project.version}</name>
|
|||
|
|
+diff --git a/broker/pom.xml b/broker/pom.xml
|
|||
|
|
+index d483e67ba..add83045d 100644
|
|||
|
|
+--- a/broker/pom.xml
|
|||
|
|
++++ b/broker/pom.xml
|
|||
|
|
+@@ -13,7 +13,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/client/pom.xml b/client/pom.xml
|
|||
|
|
+index 4febedc6d..d6fb3889b 100644
|
|||
|
|
+--- a/client/pom.xml
|
|||
|
|
++++ b/client/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/common/pom.xml b/common/pom.xml
|
|||
|
|
+index b70873dfa..6104c3ac6 100644
|
|||
|
|
+--- a/common/pom.xml
|
|||
|
|
++++ b/common/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/container/pom.xml b/container/pom.xml
|
|||
|
|
+index e6c1f4b4d..8af231e01 100644
|
|||
|
|
+--- a/container/pom.xml
|
|||
|
|
++++ b/container/pom.xml
|
|||
|
|
+@@ -18,7 +18,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/controller/pom.xml b/controller/pom.xml
|
|||
|
|
+index 46a3834c6..8432b220b 100644
|
|||
|
|
+--- a/controller/pom.xml
|
|||
|
|
++++ b/controller/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+ <packaging>jar</packaging>
|
|||
|
|
+diff --git a/distribution/pom.xml b/distribution/pom.xml
|
|||
|
|
+index 346c4de35..73474d34a 100644
|
|||
|
|
+--- a/distribution/pom.xml
|
|||
|
|
++++ b/distribution/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+ <artifactId>rocketmq-distribution</artifactId>
|
|||
|
|
+ <name>rocketmq-distribution ${project.version}</name>
|
|||
|
|
+diff --git a/example/pom.xml b/example/pom.xml
|
|||
|
|
+index 9e7db43f8..a8c7f5382 100644
|
|||
|
|
+--- a/example/pom.xml
|
|||
|
|
++++ b/example/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/filter/pom.xml b/filter/pom.xml
|
|||
|
|
+index 84189066d..892f46e9d 100644
|
|||
|
|
+--- a/filter/pom.xml
|
|||
|
|
++++ b/filter/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/namesrv/pom.xml b/namesrv/pom.xml
|
|||
|
|
+index 7c218078a..e320ed573 100644
|
|||
|
|
+--- a/namesrv/pom.xml
|
|||
|
|
++++ b/namesrv/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml
|
|||
|
|
+index fd499e3de..f10c8af6f 100644
|
|||
|
|
+--- a/openmessaging/pom.xml
|
|||
|
|
++++ b/openmessaging/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/pom.xml b/pom.xml
|
|||
|
|
+index 0e1d04f15..4202d4095 100644
|
|||
|
|
+--- a/pom.xml
|
|||
|
|
++++ b/pom.xml
|
|||
|
|
+@@ -28,7 +28,7 @@
|
|||
|
|
+ <inceptionYear>2012</inceptionYear>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ <packaging>pom</packaging>
|
|||
|
|
+ <name>Apache RocketMQ ${project.version}</name>
|
|||
|
|
+ <url>http://rocketmq.apache.org/</url>
|
|||
|
|
+@@ -37,7 +37,7 @@
|
|||
|
|
+ <url>git@github.com:apache/rocketmq.git</url>
|
|||
|
|
+ <connection>scm:git:git@github.com:apache/rocketmq.git</connection>
|
|||
|
|
+ <developerConnection>scm:git:git@github.com:apache/rocketmq.git</developerConnection>
|
|||
|
|
+- <tag>rocketmq-all-5.1.4</tag>
|
|||
|
|
++ <tag>HEAD</tag>
|
|||
|
|
+ </scm>
|
|||
|
|
+
|
|||
|
|
+ <mailingLists>
|
|||
|
|
+diff --git a/proxy/pom.xml b/proxy/pom.xml
|
|||
|
|
+index abf242eee..5c5349a8c 100644
|
|||
|
|
+--- a/proxy/pom.xml
|
|||
|
|
++++ b/proxy/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/remoting/pom.xml b/remoting/pom.xml
|
|||
|
|
+index fc70cb62e..f78480680 100644
|
|||
|
|
+--- a/remoting/pom.xml
|
|||
|
|
++++ b/remoting/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/srvutil/pom.xml b/srvutil/pom.xml
|
|||
|
|
+index d7f946cce..894e9cc6f 100644
|
|||
|
|
+--- a/srvutil/pom.xml
|
|||
|
|
++++ b/srvutil/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/store/pom.xml b/store/pom.xml
|
|||
|
|
+index 6d6983c5d..e979030e8 100644
|
|||
|
|
+--- a/store/pom.xml
|
|||
|
|
++++ b/store/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/test/pom.xml b/test/pom.xml
|
|||
|
|
+index 39090e426..168cbab0b 100644
|
|||
|
|
+--- a/test/pom.xml
|
|||
|
|
++++ b/test/pom.xml
|
|||
|
|
+@@ -20,7 +20,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml
|
|||
|
|
+index 7b209751f..b2ea40bf3 100644
|
|||
|
|
+--- a/tieredstore/pom.xml
|
|||
|
|
++++ b/tieredstore/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+diff --git a/tools/pom.xml b/tools/pom.xml
|
|||
|
|
+index 806787ec9..e1daa57a6 100644
|
|||
|
|
+--- a/tools/pom.xml
|
|||
|
|
++++ b/tools/pom.xml
|
|||
|
|
+@@ -19,7 +19,7 @@
|
|||
|
|
+ <parent>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
+ <artifactId>rocketmq-all</artifactId>
|
|||
|
|
+- <version>5.1.4</version>
|
|||
|
|
++ <version>5.1.5-SNAPSHOT</version>
|
|||
|
|
+ </parent>
|
|||
|
|
+
|
|||
|
|
+ <modelVersion>4.0.0</modelVersion>
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 88a9d939ce110381b3b418370d4711c0c214dc7f Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Ji Juntao <juntao.jjt@alibaba-inc.com>
|
|||
|
|
+Date: Sat, 23 Sep 2023 17:38:27 +0800
|
|||
|
|
+Subject: [PATCH 05/12] [ISSUE #7381] Fix the problem of inaccurate timer
|
|||
|
|
+ message metric (#7382)
|
|||
|
|
+
|
|||
|
|
+* correct the timerMetrics' result.
|
|||
|
|
+
|
|||
|
|
+* for further extension.
|
|||
|
|
+
|
|||
|
|
+* checkstyle.
|
|||
|
|
+
|
|||
|
|
+* use toLong.
|
|||
|
|
+---
|
|||
|
|
+ .../store/timer/TimerMessageStore.java | 20 +++++++++++++++----
|
|||
|
|
+ .../rocketmq/store/timer/TimerMetrics.java | 5 ++++-
|
|||
|
|
+ .../rocketmq/store/timer/TimerRequest.java | 7 +++++--
|
|||
|
|
+ .../store/timer/TimerMetricsTest.java | 10 ++++++++--
|
|||
|
|
+ 4 files changed, 33 insertions(+), 9 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
+index 0d50de65a..ac4c61cd6 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
+@@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
+ import java.util.function.Function;
|
|||
|
|
+ import org.apache.commons.collections.CollectionUtils;
|
|||
|
|
++import org.apache.commons.lang3.math.NumberUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.ServiceThread;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+ import org.apache.rocketmq.common.TopicFilterType;
|
|||
|
|
+@@ -599,7 +600,12 @@ public class TimerMessageStore {
|
|||
|
|
+ if (null == msg || null == msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC)) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+- timerMetrics.addAndGet(msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC), value);
|
|||
|
|
++ if (msg.getProperty(TIMER_ENQUEUE_MS) != null
|
|||
|
|
++ && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ // pass msg into addAndGet, for further more judgement extension.
|
|||
|
|
++ timerMetrics.addAndGet(msg, value);
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+ if (frequency.incrementAndGet() % 1000 == 0) {
|
|||
|
|
+ LOGGER.error("error in adding metric", t);
|
|||
|
|
+@@ -1323,6 +1329,7 @@ public class TimerMessageStore {
|
|||
|
|
+ perfCounterTicks.startTick(ENQUEUE_PUT);
|
|||
|
|
+ DefaultStoreMetricsManager.incTimerEnqueueCount(getRealTopic(req.getMsg()));
|
|||
|
|
+ if (shouldRunningDequeue && req.getDelayTime() < currWriteTimeMs) {
|
|||
|
|
++ req.setEnqueueTime(Long.MAX_VALUE);
|
|||
|
|
+ dequeuePutQueue.put(req);
|
|||
|
|
+ } else {
|
|||
|
|
+ boolean doEnqueueRes = doEnqueue(
|
|||
|
|
+@@ -1452,9 +1459,14 @@ public class TimerMessageStore {
|
|||
|
|
+ }
|
|||
|
|
+ try {
|
|||
|
|
+ perfCounterTicks.startTick(DEQUEUE_PUT);
|
|||
|
|
+- DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(tr.getMsg()));
|
|||
|
|
+- addMetric(tr.getMsg(), -1);
|
|||
|
|
+- MessageExtBrokerInner msg = convert(tr.getMsg(), tr.getEnqueueTime(), needRoll(tr.getMagic()));
|
|||
|
|
++ MessageExt msgExt = tr.getMsg();
|
|||
|
|
++ DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(msgExt));
|
|||
|
|
++ if (tr.getEnqueueTime() == Long.MAX_VALUE) {
|
|||
|
|
++ // never enqueue, mark it.
|
|||
|
|
++ MessageAccessor.putProperty(msgExt, TIMER_ENQUEUE_MS, String.valueOf(Long.MAX_VALUE));
|
|||
|
|
++ }
|
|||
|
|
++ addMetric(msgExt, -1);
|
|||
|
|
++ MessageExtBrokerInner msg = convert(msgExt, tr.getEnqueueTime(), needRoll(tr.getMagic()));
|
|||
|
|
+ doRes = PUT_NEED_RETRY != doPut(msg, needRoll(tr.getMagic()));
|
|||
|
|
+ while (!doRes && !isStopped()) {
|
|||
|
|
+ if (!isRunningDequeue()) {
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java
|
|||
|
|
+index e7b00cc07..7f8fedd8a 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java
|
|||
|
|
+@@ -38,6 +38,8 @@ import java.util.concurrent.locks.Lock;
|
|||
|
|
+ import java.util.concurrent.locks.ReentrantLock;
|
|||
|
|
+ import org.apache.rocketmq.common.ConfigManager;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
++import org.apache.rocketmq.common.message.MessageConst;
|
|||
|
|
++import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+@@ -78,7 +80,8 @@ public class TimerMetrics extends ConfigManager {
|
|||
|
|
+ return distPair.getCount().addAndGet(value);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public long addAndGet(String topic, int value) {
|
|||
|
|
++ public long addAndGet(MessageExt msg, int value) {
|
|||
|
|
++ String topic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC);
|
|||
|
|
+ Metric pair = getTopicPair(topic);
|
|||
|
|
+ getDataVersion().nextVersion();
|
|||
|
|
+ pair.setTimeStamp(System.currentTimeMillis());
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java
|
|||
|
|
+index 1dd64f759..1b25d355c 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java
|
|||
|
|
+@@ -27,8 +27,9 @@ public class TimerRequest {
|
|||
|
|
+ private final int sizePy;
|
|||
|
|
+ private final long delayTime;
|
|||
|
|
+
|
|||
|
|
+- private final long enqueueTime;
|
|||
|
|
+ private final int magic;
|
|||
|
|
++
|
|||
|
|
++ private long enqueueTime;
|
|||
|
|
+ private MessageExt msg;
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+@@ -94,7 +95,9 @@ public class TimerRequest {
|
|||
|
|
+ public void setLatch(CountDownLatch latch) {
|
|||
|
|
+ this.latch = latch;
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
++ public void setEnqueueTime(long enqueueTime) {
|
|||
|
|
++ this.enqueueTime = enqueueTime;
|
|||
|
|
++ }
|
|||
|
|
+ public void idempotentRelease() {
|
|||
|
|
+ idempotentRelease(true);
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java
|
|||
|
|
+index b7392cc45..3c7b9b67f 100644
|
|||
|
|
+--- a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java
|
|||
|
|
++++ b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java
|
|||
|
|
+@@ -16,6 +16,9 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.store.timer;
|
|||
|
|
+
|
|||
|
|
++import org.apache.rocketmq.common.message.MessageAccessor;
|
|||
|
|
++import org.apache.rocketmq.common.message.MessageConst;
|
|||
|
|
++import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.junit.Assert;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
+@@ -31,8 +34,11 @@ public class TimerMetricsTest {
|
|||
|
|
+
|
|||
|
|
+ TimerMetrics first = new TimerMetrics(baseDir);
|
|||
|
|
+ Assert.assertTrue(first.load());
|
|||
|
|
+- first.addAndGet("AAA", 1000);
|
|||
|
|
+- first.addAndGet("BBB", 2000);
|
|||
|
|
++ MessageExt msg = new MessageExt();
|
|||
|
|
++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "AAA");
|
|||
|
|
++ first.addAndGet(msg, 1000);
|
|||
|
|
++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "BBB");
|
|||
|
|
++ first.addAndGet(msg, 2000);
|
|||
|
|
+ Assert.assertEquals(1000, first.getTimingCount("AAA"));
|
|||
|
|
+ Assert.assertEquals(2000, first.getTimingCount("BBB"));
|
|||
|
|
+ long curr = System.currentTimeMillis();
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From d7e5c4d1a4e048cd97f0b29a96a0fc575927a03e Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: guyinyou <36399867+guyinyou@users.noreply.github.com>
|
|||
|
|
+Date: Mon, 25 Sep 2023 13:37:36 +0800
|
|||
|
|
+Subject: [PATCH 06/12] [ISSUE #7389] Fix the problem that getLastMappedFile
|
|||
|
|
+ function affects performance
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: guyinyou <guyinyou.gyy@alibaba-inc.com>
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/store/MappedFileQueue.java | 15 +++++++++++++--
|
|||
|
|
+ 1 file changed, 13 insertions(+), 2 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
|
|||
|
|
+index 32b90d14f..9a0824829 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java
|
|||
|
|
+@@ -377,8 +377,19 @@ public class MappedFileQueue implements Swappable {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public MappedFile getLastMappedFile() {
|
|||
|
|
+- MappedFile[] mappedFiles = this.mappedFiles.toArray(new MappedFile[0]);
|
|||
|
|
+- return mappedFiles.length == 0 ? null : mappedFiles[mappedFiles.length - 1];
|
|||
|
|
++ MappedFile mappedFileLast = null;
|
|||
|
|
++ while (!this.mappedFiles.isEmpty()) {
|
|||
|
|
++ try {
|
|||
|
|
++ mappedFileLast = this.mappedFiles.get(this.mappedFiles.size() - 1);
|
|||
|
|
++ break;
|
|||
|
|
++ } catch (IndexOutOfBoundsException e) {
|
|||
|
|
++ //continue;
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ log.error("getLastMappedFile has exception.", e);
|
|||
|
|
++ break;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ return mappedFileLast;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public boolean resetOffset(long offset) {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 3fd43353fdf880deb5d63ba3ad50cc6e3259dc3a Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Zhouxiang Zhan <zhouxzhan@apache.org>
|
|||
|
|
+Date: Tue, 26 Sep 2023 13:53:51 +0800
|
|||
|
|
+Subject: [PATCH 07/12] [ISSUE #7393] Add timeout configuration for grpc server
|
|||
|
|
+ (#7394)
|
|||
|
|
+
|
|||
|
|
+* Add timeout configuration for grpc server
|
|||
|
|
+
|
|||
|
|
+* Add proxyConfig
|
|||
|
|
+---
|
|||
|
|
+ .../java/org/apache/rocketmq/proxy/ProxyStartup.java | 1 +
|
|||
|
|
+ .../apache/rocketmq/proxy/config/ProxyConfig.java | 9 +++++++++
|
|||
|
|
+ .../org/apache/rocketmq/proxy/grpc/GrpcServer.java | 10 ++++++++--
|
|||
|
|
+ .../rocketmq/proxy/grpc/GrpcServerBuilder.java | 12 +++++++++++-
|
|||
|
|
+ 4 files changed, 29 insertions(+), 3 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java
|
|||
|
|
+index 06d5f4525..3b2ca99bf 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java
|
|||
|
|
+@@ -85,6 +85,7 @@ public class ProxyStartup {
|
|||
|
|
+ .addService(ChannelzService.newInstance(100))
|
|||
|
|
+ .addService(ProtoReflectionService.newInstance())
|
|||
|
|
+ .configInterceptor(accessValidators)
|
|||
|
|
++ .shutdownTime(ConfigurationManager.getProxyConfig().getGrpcShutdownTimeSeconds(), TimeUnit.SECONDS)
|
|||
|
|
+ .build();
|
|||
|
|
+ PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer);
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+index b2478fec3..c0d00d864 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
|
|||
|
|
+@@ -87,6 +87,7 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ */
|
|||
|
|
+ private String proxyMode = ProxyMode.CLUSTER.name();
|
|||
|
|
+ private Integer grpcServerPort = 8081;
|
|||
|
|
++ private long grpcShutdownTimeSeconds = 30;
|
|||
|
|
+ private int grpcBossLoopNum = 1;
|
|||
|
|
+ private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2;
|
|||
|
|
+ private boolean enableGrpcEpoll = false;
|
|||
|
|
+@@ -443,6 +444,14 @@ public class ProxyConfig implements ConfigFile {
|
|||
|
|
+ this.grpcServerPort = grpcServerPort;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public long getGrpcShutdownTimeSeconds() {
|
|||
|
|
++ return grpcShutdownTimeSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setGrpcShutdownTimeSeconds(long grpcShutdownTimeSeconds) {
|
|||
|
|
++ this.grpcShutdownTimeSeconds = grpcShutdownTimeSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public boolean isUseEndpointPortFromRequest() {
|
|||
|
|
+ return useEndpointPortFromRequest;
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java
|
|||
|
|
+index 1bffa3c0b..d5b896fe1 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java
|
|||
|
|
+@@ -29,8 +29,14 @@ public class GrpcServer implements StartAndShutdown {
|
|||
|
|
+
|
|||
|
|
+ private final Server server;
|
|||
|
|
+
|
|||
|
|
+- protected GrpcServer(Server server) {
|
|||
|
|
++ private final long timeout;
|
|||
|
|
++
|
|||
|
|
++ private final TimeUnit unit;
|
|||
|
|
++
|
|||
|
|
++ protected GrpcServer(Server server, long timeout, TimeUnit unit) {
|
|||
|
|
+ this.server = server;
|
|||
|
|
++ this.timeout = timeout;
|
|||
|
|
++ this.unit = unit;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void start() throws Exception {
|
|||
|
|
+@@ -40,7 +46,7 @@ public class GrpcServer implements StartAndShutdown {
|
|||
|
|
+
|
|||
|
|
+ public void shutdown() {
|
|||
|
|
+ try {
|
|||
|
|
+- this.server.shutdown().awaitTermination(30, TimeUnit.SECONDS);
|
|||
|
|
++ this.server.shutdown().awaitTermination(timeout, unit);
|
|||
|
|
+ log.info("grpc server shutdown successfully.");
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ e.printStackTrace();
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java
|
|||
|
|
+index 9cddd3013..0e79006f6 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java
|
|||
|
|
+@@ -41,6 +41,10 @@ public class GrpcServerBuilder {
|
|||
|
|
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
|
|||
|
|
+ protected NettyServerBuilder serverBuilder;
|
|||
|
|
+
|
|||
|
|
++ protected long time = 30;
|
|||
|
|
++
|
|||
|
|
++ protected TimeUnit unit = TimeUnit.SECONDS;
|
|||
|
|
++
|
|||
|
|
+ public static GrpcServerBuilder newBuilder(ThreadPoolExecutor executor, int port) {
|
|||
|
|
+ return new GrpcServerBuilder(executor, port);
|
|||
|
|
+ }
|
|||
|
|
+@@ -77,6 +81,12 @@ public class GrpcServerBuilder {
|
|||
|
|
+ port, bossLoopNum, workerLoopNum, maxInboundMessageSize);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public GrpcServerBuilder shutdownTime(long time, TimeUnit unit) {
|
|||
|
|
++ this.time = time;
|
|||
|
|
++ this.unit = unit;
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public GrpcServerBuilder addService(BindableService service) {
|
|||
|
|
+ this.serverBuilder.addService(service);
|
|||
|
|
+ return this;
|
|||
|
|
+@@ -93,7 +103,7 @@ public class GrpcServerBuilder {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public GrpcServer build() {
|
|||
|
|
+- return new GrpcServer(this.serverBuilder.build());
|
|||
|
|
++ return new GrpcServer(this.serverBuilder.build(), time, unit);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public GrpcServerBuilder configInterceptor(List<AccessValidator> accessValidators) {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From c3b86cd1e3c068bc5847671c899a284e49a2ecdc Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?=
|
|||
|
|
+ <shirenchuang@users.noreply.github.com>
|
|||
|
|
+Date: Tue, 26 Sep 2023 16:07:13 +0800
|
|||
|
|
+Subject: [PATCH 08/12] [ISSUE #7398] Fix ExportConfigsCommand NPE (#7399)
|
|||
|
|
+MIME-Version: 1.0
|
|||
|
|
+Content-Type: text/plain; charset=UTF-8
|
|||
|
|
+Content-Transfer-Encoding: 8bit
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: 石臻臻 <shirenchuang.src@cainiao.com>
|
|||
|
|
+---
|
|||
|
|
+ .../command/export/ExportConfigsCommand.java | 42 ++++++++++++-------
|
|||
|
|
+ 1 file changed, 26 insertions(+), 16 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java
|
|||
|
|
+index 03613b29c..c3f96d597 100644
|
|||
|
|
+--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java
|
|||
|
|
++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java
|
|||
|
|
+@@ -20,6 +20,7 @@ import java.util.HashMap;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+ import java.util.Map.Entry;
|
|||
|
|
++import java.util.Arrays;
|
|||
|
|
+ import java.util.Properties;
|
|||
|
|
+
|
|||
|
|
+ import com.alibaba.fastjson.JSON;
|
|||
|
|
+@@ -106,24 +107,33 @@ public class ExportConfigsCommand implements SubCommand {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++
|
|||
|
|
+ private Properties needBrokerProprties(Properties properties) {
|
|||
|
|
++ List<String> propertyKeys = Arrays.asList(
|
|||
|
|
++ "brokerClusterName",
|
|||
|
|
++ "brokerId",
|
|||
|
|
++ "brokerName",
|
|||
|
|
++ "brokerRole",
|
|||
|
|
++ "fileReservedTime",
|
|||
|
|
++ "filterServerNums",
|
|||
|
|
++ "flushDiskType",
|
|||
|
|
++ "maxMessageSize",
|
|||
|
|
++ "messageDelayLevel",
|
|||
|
|
++ "msgTraceTopicName",
|
|||
|
|
++ "slaveReadEnable",
|
|||
|
|
++ "traceOn",
|
|||
|
|
++ "traceTopicEnable",
|
|||
|
|
++ "useTLS",
|
|||
|
|
++ "autoCreateTopicEnable",
|
|||
|
|
++ "autoCreateSubscriptionGroup"
|
|||
|
|
++ );
|
|||
|
|
++
|
|||
|
|
+ Properties newProperties = new Properties();
|
|||
|
|
+- newProperties.setProperty("brokerClusterName", properties.getProperty("brokerClusterName"));
|
|||
|
|
+- newProperties.setProperty("brokerId", properties.getProperty("brokerId"));
|
|||
|
|
+- newProperties.setProperty("brokerName", properties.getProperty("brokerName"));
|
|||
|
|
+- newProperties.setProperty("brokerRole", properties.getProperty("brokerRole"));
|
|||
|
|
+- newProperties.setProperty("fileReservedTime", properties.getProperty("fileReservedTime"));
|
|||
|
|
+- newProperties.setProperty("filterServerNums", properties.getProperty("filterServerNums"));
|
|||
|
|
+- newProperties.setProperty("flushDiskType", properties.getProperty("flushDiskType"));
|
|||
|
|
+- newProperties.setProperty("maxMessageSize", properties.getProperty("maxMessageSize"));
|
|||
|
|
+- newProperties.setProperty("messageDelayLevel", properties.getProperty("messageDelayLevel"));
|
|||
|
|
+- newProperties.setProperty("msgTraceTopicName", properties.getProperty("msgTraceTopicName"));
|
|||
|
|
+- newProperties.setProperty("slaveReadEnable", properties.getProperty("slaveReadEnable"));
|
|||
|
|
+- newProperties.setProperty("traceOn", properties.getProperty("traceOn"));
|
|||
|
|
+- newProperties.setProperty("traceTopicEnable", properties.getProperty("traceTopicEnable"));
|
|||
|
|
+- newProperties.setProperty("useTLS", properties.getProperty("useTLS"));
|
|||
|
|
+- newProperties.setProperty("autoCreateTopicEnable", properties.getProperty("autoCreateTopicEnable"));
|
|||
|
|
+- newProperties.setProperty("autoCreateSubscriptionGroup", properties.getProperty("autoCreateSubscriptionGroup"));
|
|||
|
|
++ propertyKeys.stream()
|
|||
|
|
++ .filter(key -> properties.getProperty(key) != null)
|
|||
|
|
++ .forEach(key -> newProperties.setProperty(key, properties.getProperty(key)));
|
|||
|
|
++
|
|||
|
|
+ return newProperties;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 959a98120cea8022498557a308aff35e3d8def2a Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?=
|
|||
|
|
+ <shirenchuang@users.noreply.github.com>
|
|||
|
|
+Date: Wed, 27 Sep 2023 01:59:58 +0800
|
|||
|
|
+Subject: [PATCH 09/12] [ISSUE #7400] Fix getBrokerEpochSubCommand NPE
|
|||
|
|
+MIME-Version: 1.0
|
|||
|
|
+Content-Type: text/plain; charset=UTF-8
|
|||
|
|
+Content-Transfer-Encoding: 8bit
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: 石臻臻 <shirenchuang.src@cainiao.com>
|
|||
|
|
+---
|
|||
|
|
+ .../broker/processor/AdminBrokerProcessor.java | 10 ++++++++--
|
|||
|
|
+ 1 file changed, 8 insertions(+), 2 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
+index 9e48431be..e77120e15 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
|
|||
|
|
+@@ -2736,10 +2736,16 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
|
|||
|
|
+ final ReplicasManager replicasManager = this.brokerController.getReplicasManager();
|
|||
|
|
+ assert replicasManager != null;
|
|||
|
|
+ final BrokerConfig brokerConfig = this.brokerController.getBrokerConfig();
|
|||
|
|
++ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
++
|
|||
|
|
++ if (!brokerConfig.isEnableControllerMode()) {
|
|||
|
|
++ response.setCode(ResponseCode.SYSTEM_ERROR);
|
|||
|
|
++ response.setRemark("this request only for controllerMode ");
|
|||
|
|
++ return response;
|
|||
|
|
++ }
|
|||
|
|
+ final EpochEntryCache entryCache = new EpochEntryCache(brokerConfig.getBrokerClusterName(),
|
|||
|
|
+- brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset());
|
|||
|
|
++ brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset());
|
|||
|
|
+
|
|||
|
|
+- final RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+ response.setBody(entryCache.encode());
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+ response.setRemark(null);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 0a6ae4605fef4eaab6262fbd370aba887d8ae58b Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: tiger lee <1026203200@qq.com>
|
|||
|
|
+Date: Wed, 27 Sep 2023 14:43:15 +0800
|
|||
|
|
+Subject: [PATCH 10/12] [ISSUE #7396] Fix wrong word in
|
|||
|
|
+ BrokerController#doResterBrokerAll (#7397)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+index 53e2e1b62..d4bded600 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
|
|||
|
|
+@@ -1807,7 +1807,7 @@ public class BrokerController {
|
|||
|
|
+ TopicConfigSerializeWrapper topicConfigWrapper) {
|
|||
|
|
+
|
|||
|
|
+ if (shutdown) {
|
|||
|
|
+- BrokerController.LOG.info("BrokerController#doResterBrokerAll: broker has shutdown, no need to register any more.");
|
|||
|
|
++ BrokerController.LOG.info("BrokerController#doRegisterBrokerAll: broker has shutdown, no need to register any more.");
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ List<RegisterBrokerResult> registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll(
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 4f1b42a7c5557bcadd6b9982a0c9bd876622c69e Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: ShuangxiDing <dingshuangxi888@gmail.com>
|
|||
|
|
+Date: Thu, 28 Sep 2023 16:52:02 +0800
|
|||
|
|
+Subject: [PATCH 11/12] [ISSUE #7410] Handle the Exception when the Proxy
|
|||
|
|
+ requests the client
|
|||
|
|
+MIME-Version: 1.0
|
|||
|
|
+Content-Type: text/plain; charset=UTF-8
|
|||
|
|
+Content-Transfer-Encoding: 8bit
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: 徒钟 <shuangxi.dsx@alibaba-inc.com>
|
|||
|
|
+---
|
|||
|
|
+ .../remoting/channel/RemotingChannel.java | 23 ++++++++++++++-----
|
|||
|
|
+ 1 file changed, 17 insertions(+), 6 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
|
|||
|
|
+index 40946cabf..d755fdcc4 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
|
|||
|
|
+@@ -34,6 +34,7 @@ import org.apache.rocketmq.common.utils.NetworkUtil;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
|
|||
|
|
++import org.apache.rocketmq.proxy.common.utils.ExceptionUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.common.utils.FutureUtils;
|
|||
|
|
+ import org.apache.rocketmq.proxy.config.ConfigurationManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter;
|
|||
|
|
+@@ -158,10 +159,15 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver
|
|||
|
|
+ if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
+ ConsumerRunningInfo consumerRunningInfo = ConsumerRunningInfo.decode(response.getBody(), ConsumerRunningInfo.class);
|
|||
|
|
+ responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", consumerRunningInfo));
|
|||
|
|
++ } else {
|
|||
|
|
++ String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark());
|
|||
|
|
++ RuntimeException e = new RuntimeException(errMsg);
|
|||
|
|
++ responseFuture.completeExceptionally(e);
|
|||
|
|
+ }
|
|||
|
|
+- String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark());
|
|||
|
|
+- RuntimeException e = new RuntimeException(errMsg);
|
|||
|
|
+- responseFuture.completeExceptionally(e);
|
|||
|
|
++ })
|
|||
|
|
++ .exceptionally(t -> {
|
|||
|
|
++ responseFuture.completeExceptionally(ExceptionUtils.getRealException(t));
|
|||
|
|
++ return null;
|
|||
|
|
+ });
|
|||
|
|
+ return CompletableFuture.completedFuture(null);
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+@@ -183,10 +189,15 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver
|
|||
|
|
+ if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
+ ConsumeMessageDirectlyResult result = ConsumeMessageDirectlyResult.decode(response.getBody(), ConsumeMessageDirectlyResult.class);
|
|||
|
|
+ responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", result));
|
|||
|
|
++ } else {
|
|||
|
|
++ String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark());
|
|||
|
|
++ RuntimeException e = new RuntimeException(errMsg);
|
|||
|
|
++ responseFuture.completeExceptionally(e);
|
|||
|
|
+ }
|
|||
|
|
+- String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark());
|
|||
|
|
+- RuntimeException e = new RuntimeException(errMsg);
|
|||
|
|
+- responseFuture.completeExceptionally(e);
|
|||
|
|
++ })
|
|||
|
|
++ .exceptionally(t -> {
|
|||
|
|
++ responseFuture.completeExceptionally(ExceptionUtils.getRealException(t));
|
|||
|
|
++ return null;
|
|||
|
|
+ });
|
|||
|
|
+ return CompletableFuture.completedFuture(null);
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From c36bb78e850129b9db40adc5b0e1b9bfd5c8fd2e Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: shriVATSA54 <116296557+shriVATSA54@users.noreply.github.com>
|
|||
|
|
+Date: Sat, 7 Oct 2023 12:22:39 +0530
|
|||
|
|
+Subject: [PATCH 12/12] [ISSUE 7313] Enhancement Optimization Method name
|
|||
|
|
+ (#7420)
|
|||
|
|
+
|
|||
|
|
+* Enhancment/method_name/#7313/
|
|||
|
|
+
|
|||
|
|
+* Enhancment/method_name/#7313/
|
|||
|
|
+
|
|||
|
|
+* Enhancment/method_name/#7313/
|
|||
|
|
+---
|
|||
|
|
+ .../tieredstore/provider/TieredStoreTopicBlackListFilter.java | 2 +-
|
|||
|
|
+ .../rocketmq/tieredstore/provider/TieredStoreTopicFilter.java | 2 +-
|
|||
|
|
+ .../provider/TieredStoreTopicBlackListFilterTest.java | 2 +-
|
|||
|
|
+ 3 files changed, 3 insertions(+), 3 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java
|
|||
|
|
+index 50adbb713..f8bf165bc 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java
|
|||
|
|
+@@ -39,7 +39,7 @@ public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+- public void addTopicToWhiteList(String topicName) {
|
|||
|
|
++ public void addTopicToBlackList(String topicName) {
|
|||
|
|
+ this.topicBlackSet.add(topicName);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java
|
|||
|
|
+index 3f26b8b02..f983ed6e9 100644
|
|||
|
|
+--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java
|
|||
|
|
++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java
|
|||
|
|
+@@ -21,5 +21,5 @@ public interface TieredStoreTopicFilter {
|
|||
|
|
+
|
|||
|
|
+ boolean filterTopic(String topicName);
|
|||
|
|
+
|
|||
|
|
+- void addTopicToWhiteList(String topicName);
|
|||
|
|
++ void addTopicToBlackList(String topicName);
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java
|
|||
|
|
+index 2bf48173c..fbaafa1b4 100644
|
|||
|
|
+--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java
|
|||
|
|
++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java
|
|||
|
|
+@@ -30,7 +30,7 @@ public class TieredStoreTopicBlackListFilterTest {
|
|||
|
|
+
|
|||
|
|
+ String topicName = "WhiteTopic";
|
|||
|
|
+ Assert.assertFalse(topicFilter.filterTopic(topicName));
|
|||
|
|
+- topicFilter.addTopicToWhiteList(topicName);
|
|||
|
|
++ topicFilter.addTopicToBlackList(topicName);
|
|||
|
|
+ Assert.assertTrue(topicFilter.filterTopic(topicName));
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+\ No newline at end of file
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch020-backport-add-goaway-mechanism.patch b/patch020-backport-add-goaway-mechanism.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..122db0325
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch020-backport-add-goaway-mechanism.patch
|
|||
|
|
@@ -0,0 +1,3696 @@
|
|||
|
|
+From 84156084a4c5228e1d2fe21e068fff330bbc40d1 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Zhouxiang Zhan <zhouxzhan@apache.org>
|
|||
|
|
+Date: Sun, 8 Oct 2023 11:13:25 +0800
|
|||
|
|
+Subject: [PATCH 1/7] [ISSUE #7321] Refector NettyRemotingAbstract with unify
|
|||
|
|
+ future implementation (#7322)
|
|||
|
|
+
|
|||
|
|
+* Refector NettyRemotingAbstract
|
|||
|
|
+
|
|||
|
|
+* Add invoke with future method
|
|||
|
|
+
|
|||
|
|
+* Deprecate InvokeCallback#operationComplete
|
|||
|
|
+
|
|||
|
|
+* Add operationSuccess and operationException for InvokeCallback
|
|||
|
|
+
|
|||
|
|
+* fix unit test
|
|||
|
|
+
|
|||
|
|
+* fix unit test
|
|||
|
|
+
|
|||
|
|
+* Keep InvokeCallback#operationComplete
|
|||
|
|
+
|
|||
|
|
+* Optimize invokeAsyncImpl operationComplete
|
|||
|
|
+
|
|||
|
|
+* Add unit test for NettyRemotingClient
|
|||
|
|
+
|
|||
|
|
+* fix checkstyle
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/broker/out/BrokerOuterAPI.java | 147 +++++----
|
|||
|
|
+ .../rocketmq/client/impl/MQAdminImpl.java | 71 ++--
|
|||
|
|
+ .../rocketmq/client/impl/MQClientAPIImpl.java | 239 +++++++-------
|
|||
|
|
+ .../client/impl/mqclient/MQClientAPIExt.java | 309 ++++++++----------
|
|||
|
|
+ .../client/impl/MQClientAPIImplTest.java | 12 +-
|
|||
|
|
+ .../remoting/RemotingProtocolServer.java | 22 +-
|
|||
|
|
+ .../service/mqclient/MQClientAPIExtTest.java | 97 +++---
|
|||
|
|
+ .../rocketmq/remoting/InvokeCallback.java | 15 +
|
|||
|
|
+ .../rocketmq/remoting/RemotingClient.java | 27 +-
|
|||
|
|
+ .../remoting/netty/NettyRemotingAbstract.java | 123 ++++---
|
|||
|
|
+ .../remoting/netty/NettyRemotingClient.java | 33 +-
|
|||
|
|
+ .../remoting/netty/ResponseFuture.java | 15 +
|
|||
|
|
+ .../rocketmq/remoting/rpc/RpcClientImpl.java | 29 +-
|
|||
|
|
+ .../rocketmq/remoting/RemotingServerTest.java | 22 +-
|
|||
|
|
+ .../rocketmq/remoting/netty/MockChannel.java | 21 +-
|
|||
|
|
+ .../remoting/netty/MockChannelPromise.java | 191 +++++++++++
|
|||
|
|
+ .../netty/NettyRemotingAbstractTest.java | 54 ++-
|
|||
|
|
+ .../netty/NettyRemotingClientTest.java | 185 ++++++++++-
|
|||
|
|
+ 18 files changed, 1029 insertions(+), 583 deletions(-)
|
|||
|
|
+ rename client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java => remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java (57%)
|
|||
|
|
+ create mode 100644 remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
|
|||
|
|
+index 9dfb8127d..6fde48dd9 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
|
|||
|
|
+@@ -73,6 +73,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyClientConfig;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
|
|||
|
|
++import org.apache.rocketmq.remoting.netty.ResponseFuture;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.DataVersion;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+@@ -107,6 +108,8 @@ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;
|
|||
|
|
+@@ -124,8 +127,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerReques
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterTopicRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;
|
|||
|
|
+-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.BrokerData;
|
|||
|
|
+@@ -151,7 +152,6 @@ public class BrokerOuterAPI {
|
|||
|
|
+ private final RpcClient rpcClient;
|
|||
|
|
+ private String nameSrvAddr = null;
|
|||
|
|
+
|
|||
|
|
+-
|
|||
|
|
+ public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) {
|
|||
|
|
+ this(nettyClientConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata());
|
|||
|
|
+ }
|
|||
|
|
+@@ -459,7 +459,7 @@ public class BrokerOuterAPI {
|
|||
|
|
+ * @param filterServerList
|
|||
|
|
+ * @param oneway
|
|||
|
|
+ * @param timeoutMills
|
|||
|
|
+- * @param compressed default false
|
|||
|
|
++ * @param compressed default false
|
|||
|
|
+ * @return
|
|||
|
|
+ */
|
|||
|
|
+ public List<RegisterBrokerResult> registerBrokerAll(
|
|||
|
|
+@@ -643,7 +643,6 @@ public class BrokerOuterAPI {
|
|||
|
|
+ queueDatas.add(queueData);
|
|||
|
|
+ final byte[] topicRouteBody = topicRouteData.encode();
|
|||
|
|
+
|
|||
|
|
+-
|
|||
|
|
+ List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();
|
|||
|
|
+ final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());
|
|||
|
|
+ for (final String namesrvAddr : nameServerAddressList) {
|
|||
|
|
+@@ -910,25 +909,33 @@ public class BrokerOuterAPI {
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null);
|
|||
|
|
+
|
|||
|
|
+ request.setBody(requestBody.encode());
|
|||
|
|
+- this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- if (callback == null) {
|
|||
|
|
+- return;
|
|||
|
|
++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- try {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
+- LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(),
|
|||
|
|
+- LockBatchResponseBody.class);
|
|||
|
|
+- Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
|
|||
|
|
+- callback.onSuccess(messageQueues);
|
|||
|
|
+- } else {
|
|||
|
|
+- callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
+- }
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ if (callback == null) {
|
|||
|
|
++ return;
|
|||
|
|
+ }
|
|||
|
|
+- } catch (Throwable ignored) {
|
|||
|
|
++ if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
++ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(),
|
|||
|
|
++ LockBatchResponseBody.class);
|
|||
|
|
++ Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
|
|||
|
|
++ callback.onSuccess(messageQueues);
|
|||
|
|
++ } else {
|
|||
|
|
++ callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ if (callback == null) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ callback.onException(throwable);
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+@@ -942,22 +949,30 @@ public class BrokerOuterAPI {
|
|||
|
|
+
|
|||
|
|
+ request.setBody(requestBody.encode());
|
|||
|
|
+
|
|||
|
|
+- this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- if (callback == null) {
|
|||
|
|
+- return;
|
|||
|
|
++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- try {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
+- callback.onSuccess();
|
|||
|
|
+- } else {
|
|||
|
|
+- callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
+- }
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ if (callback == null) {
|
|||
|
|
++ return;
|
|||
|
|
+ }
|
|||
|
|
+- } catch (Throwable ignored) {
|
|||
|
|
++ if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
++ callback.onSuccess();
|
|||
|
|
++ } else {
|
|||
|
|
++ callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ if (callback == null) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ callback.onException(throwable);
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+@@ -983,21 +998,27 @@ public class BrokerOuterAPI {
|
|||
|
|
+ CompletableFuture<SendResult> cf = new CompletableFuture<>();
|
|||
|
|
+ final String msgId = msg.getMsgId();
|
|||
|
|
+ try {
|
|||
|
|
+- this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (null != response) {
|
|||
|
|
+- SendResult sendResult = null;
|
|||
|
|
++ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ try {
|
|||
|
|
+- sendResult = this.processSendResponse(brokerName, msg, response);
|
|||
|
|
++ SendResult sendResult = processSendResponse(brokerName, msg, response);
|
|||
|
|
+ cf.complete(sendResult);
|
|||
|
|
+ } catch (MQBrokerException | RemotingCommandException e) {
|
|||
|
|
+ LOGGER.error("processSendResponse in sendMessageToSpecificBrokerAsync failed, msgId=" + msgId, e);
|
|||
|
|
+ cf.completeExceptionally(e);
|
|||
|
|
+ }
|
|||
|
|
+- } else {
|
|||
|
|
+- cf.complete(null);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ cf.completeExceptionally(throwable);
|
|||
|
|
++ }
|
|||
|
|
+ });
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+ LOGGER.error("invokeAsync failed in sendMessageToSpecificBrokerAsync, msgId=" + msgId, t);
|
|||
|
|
+@@ -1057,7 +1078,7 @@ public class BrokerOuterAPI {
|
|||
|
|
+ }
|
|||
|
|
+ if (sendStatus != null) {
|
|||
|
|
+ SendMessageResponseHeader responseHeader =
|
|||
|
|
+- (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
|
|||
|
|
++ (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
|
|||
|
|
+
|
|||
|
|
+ //If namespace not null , reset Topic without namespace.
|
|||
|
|
+ String topic = msg.getTopic();
|
|||
|
|
+@@ -1073,8 +1094,8 @@ public class BrokerOuterAPI {
|
|||
|
|
+ uniqMsgId = sb.toString();
|
|||
|
|
+ }
|
|||
|
|
+ SendResult sendResult = new SendResult(sendStatus,
|
|||
|
|
+- uniqMsgId,
|
|||
|
|
+- responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());
|
|||
|
|
++ uniqMsgId,
|
|||
|
|
++ responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());
|
|||
|
|
+ sendResult.setTransactionId(responseHeader.getTransactionId());
|
|||
|
|
+ String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);
|
|||
|
|
+ String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH);
|
|||
|
|
+@@ -1218,8 +1239,9 @@ public class BrokerOuterAPI {
|
|||
|
|
+ /**
|
|||
|
|
+ * Broker try to elect itself as a master in broker set
|
|||
|
|
+ */
|
|||
|
|
+- public Pair<ElectMasterResponseHeader, Set<Long>> brokerElect(String controllerAddress, String clusterName, String brokerName,
|
|||
|
|
+- Long brokerId) throws Exception {
|
|||
|
|
++ public Pair<ElectMasterResponseHeader, Set<Long>> brokerElect(String controllerAddress, String clusterName,
|
|||
|
|
++ String brokerName,
|
|||
|
|
++ Long brokerId) throws Exception {
|
|||
|
|
+
|
|||
|
|
+ final ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId);
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, requestHeader);
|
|||
|
|
+@@ -1237,7 +1259,8 @@ public class BrokerOuterAPI {
|
|||
|
|
+ throw new MQBrokerException(response.getCode(), response.getRemark());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, final String controllerAddress) throws Exception {
|
|||
|
|
++ public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName,
|
|||
|
|
++ final String controllerAddress) throws Exception {
|
|||
|
|
+ final GetNextBrokerIdRequestHeader requestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName);
|
|||
|
|
+ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, requestHeader);
|
|||
|
|
+ final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
|
|||
|
|
+@@ -1248,7 +1271,8 @@ public class BrokerOuterAPI {
|
|||
|
|
+ throw new MQBrokerException(response.getCode(), response.getRemark());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception {
|
|||
|
|
++ public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName,
|
|||
|
|
++ final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception {
|
|||
|
|
+ final ApplyBrokerIdRequestHeader requestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, brokerId, registerCheckCode);
|
|||
|
|
+ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, requestHeader);
|
|||
|
|
+ final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
|
|||
|
|
+@@ -1259,7 +1283,9 @@ public class BrokerOuterAPI {
|
|||
|
|
+ throw new MQBrokerException(response.getCode(), response.getRemark());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public Pair<RegisterBrokerToControllerResponseHeader, Set<Long>> registerBrokerToController(final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, final String controllerAddress) throws Exception {
|
|||
|
|
++ public Pair<RegisterBrokerToControllerResponseHeader, Set<Long>> registerBrokerToController(
|
|||
|
|
++ final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress,
|
|||
|
|
++ final String controllerAddress) throws Exception {
|
|||
|
|
+ final RegisterBrokerToControllerRequestHeader requestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, brokerId, brokerAddress);
|
|||
|
|
+ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, requestHeader);
|
|||
|
|
+ final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
|
|||
|
|
+@@ -1355,16 +1381,25 @@ public class BrokerOuterAPI {
|
|||
|
|
+
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);
|
|||
|
|
+ CompletableFuture<PullResult> pullResultFuture = new CompletableFuture<>();
|
|||
|
|
+- this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- if (responseFuture.getCause() != null) {
|
|||
|
|
+- pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
|
|||
|
|
+- return;
|
|||
|
|
++ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
+ }
|
|||
|
|
+- try {
|
|||
|
|
+- PullResultExt pullResultExt = this.processPullResponse(responseFuture.getResponseCommand(), brokerAddr);
|
|||
|
|
+- this.processPullResult(pullResultExt, brokerName, queueId);
|
|||
|
|
+- pullResultFuture.complete(pullResultExt);
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ try {
|
|||
|
|
++ PullResultExt pullResultExt = processPullResponse(response, brokerAddr);
|
|||
|
|
++ processPullResult(pullResultExt, brokerName, queueId);
|
|||
|
|
++ pullResultFuture.complete(pullResultExt);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
+ pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
|
|||
|
|
+index 1ef3a9483..83835bd3d 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
|
|||
|
|
+@@ -44,6 +44,8 @@ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageId;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.common.utils.NetworkUtil;
|
|||
|
|
++import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+ import org.apache.rocketmq.remoting.InvokeCallback;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingCommandException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingException;
|
|||
|
|
+@@ -55,8 +57,6 @@ import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.BrokerData;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
|
|||
|
|
+-import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+
|
|||
|
|
+ public class MQAdminImpl {
|
|||
|
|
+
|
|||
|
|
+@@ -357,44 +357,51 @@ public class MQAdminImpl {
|
|||
|
|
+ new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ try {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- switch (response.getCode()) {
|
|||
|
|
+- case ResponseCode.SUCCESS: {
|
|||
|
|
+- QueryMessageResponseHeader responseHeader = null;
|
|||
|
|
+- try {
|
|||
|
|
+- responseHeader =
|
|||
|
|
+- (QueryMessageResponseHeader) response
|
|||
|
|
+- .decodeCommandCustomHeader(QueryMessageResponseHeader.class);
|
|||
|
|
+- } catch (RemotingCommandException e) {
|
|||
|
|
+- log.error("decodeCommandCustomHeader exception", e);
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- List<MessageExt> wrappers =
|
|||
|
|
+- MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true);
|
|||
|
|
+-
|
|||
|
|
+- QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers);
|
|||
|
|
+- try {
|
|||
|
|
+- lock.writeLock().lock();
|
|||
|
|
+- queryResultList.add(qr);
|
|||
|
|
+- } finally {
|
|||
|
|
+- lock.writeLock().unlock();
|
|||
|
|
+- }
|
|||
|
|
+- break;
|
|||
|
|
++ switch (response.getCode()) {
|
|||
|
|
++ case ResponseCode.SUCCESS: {
|
|||
|
|
++ QueryMessageResponseHeader responseHeader = null;
|
|||
|
|
++ try {
|
|||
|
|
++ responseHeader =
|
|||
|
|
++ (QueryMessageResponseHeader) response
|
|||
|
|
++ .decodeCommandCustomHeader(QueryMessageResponseHeader.class);
|
|||
|
|
++ } catch (RemotingCommandException e) {
|
|||
|
|
++ log.error("decodeCommandCustomHeader exception", e);
|
|||
|
|
++ return;
|
|||
|
|
+ }
|
|||
|
|
+- default:
|
|||
|
|
+- log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark());
|
|||
|
|
+- break;
|
|||
|
|
++
|
|||
|
|
++ List<MessageExt> wrappers =
|
|||
|
|
++ MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true);
|
|||
|
|
++
|
|||
|
|
++ QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers);
|
|||
|
|
++ try {
|
|||
|
|
++ lock.writeLock().lock();
|
|||
|
|
++ queryResultList.add(qr);
|
|||
|
|
++ } finally {
|
|||
|
|
++ lock.writeLock().unlock();
|
|||
|
|
++ }
|
|||
|
|
++ break;
|
|||
|
|
+ }
|
|||
|
|
+- } else {
|
|||
|
|
+- log.warn("getResponseCommand return null");
|
|||
|
|
++ default:
|
|||
|
|
++ log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark());
|
|||
|
|
++ break;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ } finally {
|
|||
|
|
+ countDownLatch.countDown();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ log.error("queryMessage error, requestHeader={}", requestHeader);
|
|||
|
|
++ countDownLatch.countDown();
|
|||
|
|
++ }
|
|||
|
|
+ }, isUniqKey);
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ log.warn("queryMessage exception", e);
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+index 3201a493f..2407e5737 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+@@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.rocketmq.client.ClientConfig;
|
|||
|
|
+ import org.apache.rocketmq.client.Validators;
|
|||
|
|
+-import org.apache.rocketmq.client.common.ClientErrorCode;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckCallback;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckResult;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.AckStatus;
|
|||
|
|
+@@ -653,10 +652,13 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
+- long cost = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (null == sendCallback && response != null) {
|
|||
|
|
+
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ long cost = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
++ if (null == sendCallback) {
|
|||
|
|
+ try {
|
|||
|
|
+ SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr);
|
|||
|
|
+ if (context != null && sendResult != null) {
|
|||
|
|
+@@ -666,46 +668,47 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ } catch (Throwable e) {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true);
|
|||
|
|
++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true);
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- if (response != null) {
|
|||
|
|
++ try {
|
|||
|
|
++ SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr);
|
|||
|
|
++ assert sendResult != null;
|
|||
|
|
++ if (context != null) {
|
|||
|
|
++ context.setSendResult(sendResult);
|
|||
|
|
++ context.getProducer().executeSendMessageHookAfter(context);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ try {
|
|||
|
|
+- SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr);
|
|||
|
|
+- assert sendResult != null;
|
|||
|
|
+- if (context != null) {
|
|||
|
|
+- context.setSendResult(sendResult);
|
|||
|
|
+- context.getProducer().executeSendMessageHookAfter(context);
|
|||
|
|
+- }
|
|||
|
|
++ sendCallback.onSuccess(sendResult);
|
|||
|
|
++ } catch (Throwable e) {
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
+- try {
|
|||
|
|
+- sendCallback.onSuccess(sendResult);
|
|||
|
|
+- } catch (Throwable e) {
|
|||
|
|
+- }
|
|||
|
|
++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true);
|
|||
|
|
++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
++ retryTimesWhenSendFailed, times, e, context, false, producer);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
+- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true);
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true);
|
|||
|
|
+- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
+- retryTimesWhenSendFailed, times, e, context, false, producer);
|
|||
|
|
+- }
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true);
|
|||
|
|
++ long cost = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
++ if (throwable instanceof RemotingSendRequestException) {
|
|||
|
|
++ MQClientException ex = new MQClientException("send request failed", throwable);
|
|||
|
|
++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
++ retryTimesWhenSendFailed, times, ex, context, true, producer);
|
|||
|
|
++ } else if (throwable instanceof RemotingTimeoutException) {
|
|||
|
|
++ MQClientException ex = new MQClientException("wait response timeout, cost=" + cost, throwable);
|
|||
|
|
++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
++ retryTimesWhenSendFailed, times, ex, context, true, producer);
|
|||
|
|
+ } else {
|
|||
|
|
+- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true);
|
|||
|
|
+- if (!responseFuture.isSendRequestOK()) {
|
|||
|
|
+- MQClientException ex = new MQClientException("send request failed", responseFuture.getCause());
|
|||
|
|
+- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
+- retryTimesWhenSendFailed, times, ex, context, true, producer);
|
|||
|
|
+- } else if (responseFuture.isTimeout()) {
|
|||
|
|
+- MQClientException ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms",
|
|||
|
|
+- responseFuture.getCause());
|
|||
|
|
+- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
+- retryTimesWhenSendFailed, times, ex, context, true, producer);
|
|||
|
|
+- } else {
|
|||
|
|
+- MQClientException ex = new MQClientException("unknow reseaon", responseFuture.getCause());
|
|||
|
|
+- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
+- retryTimesWhenSendFailed, times, ex, context, true, producer);
|
|||
|
|
+- }
|
|||
|
|
++ MQClientException ex = new MQClientException("unknow reseaon", throwable);
|
|||
|
|
++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
|
|||
|
|
++ retryTimesWhenSendFailed, times, ex, context, true, producer);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+@@ -857,30 +860,25 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ final long timeoutMillis, final PopCallback popCallback
|
|||
|
|
+ ) throws RemotingException, InterruptedException {
|
|||
|
|
+ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader);
|
|||
|
|
+- this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) {
|
|||
|
|
++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+- public void onComplete(ResponseFuture responseFuture) {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- try {
|
|||
|
|
+- PopResult
|
|||
|
|
+- popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader);
|
|||
|
|
+- assert popResult != null;
|
|||
|
|
+- popCallback.onSuccess(popResult);
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- popCallback.onException(e);
|
|||
|
|
+- }
|
|||
|
|
+- } else {
|
|||
|
|
+- if (!responseFuture.isSendRequestOK()) {
|
|||
|
|
+- popCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
|
|||
|
|
+- } else if (responseFuture.isTimeout()) {
|
|||
|
|
+- popCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
|
|||
|
|
+- responseFuture.getCause()));
|
|||
|
|
+- } else {
|
|||
|
|
+- popCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
|
|||
|
|
+- }
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ try {
|
|||
|
|
++ PopResult popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader);
|
|||
|
|
++ popCallback.onSuccess(popResult);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ popCallback.onException(e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ popCallback.onException(throwable);
|
|||
|
|
++ }
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -959,34 +957,26 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ request.setBody(requestBody.encode());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+- this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) {
|
|||
|
|
++ this.remotingClient.invokeAsync(addr, request, timeOut, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+- public void onComplete(ResponseFuture responseFuture) {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- try {
|
|||
|
|
+- AckResult ackResult = new AckResult();
|
|||
|
|
+- if (ResponseCode.SUCCESS == response.getCode()) {
|
|||
|
|
+- ackResult.setStatus(AckStatus.OK);
|
|||
|
|
+- } else {
|
|||
|
|
+- ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
+- }
|
|||
|
|
+- ackCallback.onSuccess(ackResult);
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- ackCallback.onException(e);
|
|||
|
|
+- }
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ if (ResponseCode.SUCCESS == response.getCode()) {
|
|||
|
|
++ ackResult.setStatus(AckStatus.OK);
|
|||
|
|
+ } else {
|
|||
|
|
+- if (!responseFuture.isSendRequestOK()) {
|
|||
|
|
+- ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
|
|||
|
|
+- } else if (responseFuture.isTimeout()) {
|
|||
|
|
+- ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
|
|||
|
|
+- responseFuture.getCause()));
|
|||
|
|
+- } else {
|
|||
|
|
+- ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeOut + ". Request: " + request, responseFuture.getCause()));
|
|||
|
|
+- }
|
|||
|
|
++ ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
+ }
|
|||
|
|
++ ackCallback.onSuccess(ackResult);
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ ackCallback.onException(throwable);
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+@@ -999,39 +989,37 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ final AckCallback ackCallback
|
|||
|
|
+ ) throws RemotingException, MQBrokerException, InterruptedException {
|
|||
|
|
+ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);
|
|||
|
|
+- this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) {
|
|||
|
|
++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+- public void onComplete(ResponseFuture responseFuture) {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- try {
|
|||
|
|
+- ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class);
|
|||
|
|
+- AckResult ackResult = new AckResult();
|
|||
|
|
+- if (ResponseCode.SUCCESS == response.getCode()) {
|
|||
|
|
+- ackResult.setStatus(AckStatus.OK);
|
|||
|
|
+- ackResult.setPopTime(responseHeader.getPopTime());
|
|||
|
|
+- ackResult.setExtraInfo(ExtraInfoUtil
|
|||
|
|
+- .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(),
|
|||
|
|
+- responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR
|
|||
|
|
+- + requestHeader.getOffset());
|
|||
|
|
+- } else {
|
|||
|
|
+- ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
+- }
|
|||
|
|
+- ackCallback.onSuccess(ackResult);
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- ackCallback.onException(e);
|
|||
|
|
+- }
|
|||
|
|
+- } else {
|
|||
|
|
+- if (!responseFuture.isSendRequestOK()) {
|
|||
|
|
+- ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
|
|||
|
|
+- } else if (responseFuture.isTimeout()) {
|
|||
|
|
+- ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
|
|||
|
|
+- responseFuture.getCause()));
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ try {
|
|||
|
|
++ ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class);
|
|||
|
|
++ AckResult ackResult = new AckResult();
|
|||
|
|
++ if (ResponseCode.SUCCESS == response.getCode()) {
|
|||
|
|
++ ackResult.setStatus(AckStatus.OK);
|
|||
|
|
++ ackResult.setPopTime(responseHeader.getPopTime());
|
|||
|
|
++ ackResult.setExtraInfo(ExtraInfoUtil
|
|||
|
|
++ .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(),
|
|||
|
|
++ responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR
|
|||
|
|
++ + requestHeader.getOffset());
|
|||
|
|
+ } else {
|
|||
|
|
+- ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
|
|||
|
|
++ ackResult.setStatus(AckStatus.NO_EXIST);
|
|||
|
|
+ }
|
|||
|
|
++ ackCallback.onSuccess(ackResult);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ ackCallback.onException(e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ ackCallback.onException(throwable);
|
|||
|
|
++ }
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -1044,26 +1032,23 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- try {
|
|||
|
|
+- PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr);
|
|||
|
|
+- assert pullResult != null;
|
|||
|
|
+- pullCallback.onSuccess(pullResult);
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- pullCallback.onException(e);
|
|||
|
|
+- }
|
|||
|
|
+- } else {
|
|||
|
|
+- if (!responseFuture.isSendRequestOK()) {
|
|||
|
|
+- pullCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
|
|||
|
|
+- } else if (responseFuture.isTimeout()) {
|
|||
|
|
+- pullCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
|
|||
|
|
+- responseFuture.getCause()));
|
|||
|
|
+- } else {
|
|||
|
|
+- pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
|
|||
|
|
+- }
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ try {
|
|||
|
|
++ PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr);
|
|||
|
|
++ pullCallback.onSuccess(pullResult);
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ pullCallback.onException(e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ pullCallback.onException(throwable);
|
|||
|
|
++ }
|
|||
|
|
+ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
|
|||
|
|
+index d7c8ef8d9..f3102e175 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
|
|||
|
|
+@@ -30,7 +30,6 @@ import org.apache.rocketmq.client.consumer.PullCallback;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.PullResult;
|
|||
|
|
+ import org.apache.rocketmq.client.consumer.PullStatus;
|
|||
|
|
+ import org.apache.rocketmq.client.exception.MQBrokerException;
|
|||
|
|
+-import org.apache.rocketmq.client.exception.MQClientException;
|
|||
|
|
+ import org.apache.rocketmq.client.exception.OffsetNotFoundException;
|
|||
|
|
+ import org.apache.rocketmq.client.impl.ClientRemotingProcessor;
|
|||
|
|
+ import org.apache.rocketmq.client.impl.CommunicationMode;
|
|||
|
|
+@@ -47,6 +46,7 @@ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
++import org.apache.rocketmq.remoting.InvokeCallback;
|
|||
|
|
+ import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingCommandException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyClientConfig;
|
|||
|
|
+@@ -106,19 +106,6 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- protected static MQClientException processNullResponseErr(ResponseFuture responseFuture) {
|
|||
|
|
+- MQClientException ex;
|
|||
|
|
+- if (!responseFuture.isSendRequestOK()) {
|
|||
|
|
+- ex = new MQClientException("send request failed", responseFuture.getCause());
|
|||
|
|
+- } else if (responseFuture.isTimeout()) {
|
|||
|
|
+- ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms",
|
|||
|
|
+- responseFuture.getCause());
|
|||
|
|
+- } else {
|
|||
|
|
+- ex = new MQClientException("unknown reason", responseFuture.getCause());
|
|||
|
|
+- }
|
|||
|
|
+- return ex;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+ public CompletableFuture<Void> sendHeartbeatOneway(
|
|||
|
|
+ String brokerAddr,
|
|||
|
|
+ HeartbeatData heartbeatData,
|
|||
|
|
+@@ -146,24 +133,15 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ request.setLanguage(clientConfig.getLanguage());
|
|||
|
|
+ request.setBody(heartbeatData.encode());
|
|||
|
|
+
|
|||
|
|
+- CompletableFuture<Integer> future = new CompletableFuture<>();
|
|||
|
|
+- try {
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- if (ResponseCode.SUCCESS == response.getCode()) {
|
|||
|
|
+- future.complete(response.getVersion());
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr));
|
|||
|
|
+- }
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
+- }
|
|||
|
|
+- });
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
+- }
|
|||
|
|
+- return future;
|
|||
|
|
++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
|
|||
|
|
++ CompletableFuture<Integer> future0 = new CompletableFuture<>();
|
|||
|
|
++ if (ResponseCode.SUCCESS == response.getCode()) {
|
|||
|
|
++ future0.complete(response.getVersion());
|
|||
|
|
++ } else {
|
|||
|
|
++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr));
|
|||
|
|
++ }
|
|||
|
|
++ return future0;
|
|||
|
|
++ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CompletableFuture<SendResult> sendMessageAsync(
|
|||
|
|
+@@ -177,24 +155,15 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2);
|
|||
|
|
+ request.setBody(msg.getBody());
|
|||
|
|
+
|
|||
|
|
+- CompletableFuture<SendResult> future = new CompletableFuture<>();
|
|||
|
|
+- try {
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- try {
|
|||
|
|
+- future.complete(this.processSendResponse(brokerName, msg, response, brokerAddr));
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- future.completeExceptionally(e);
|
|||
|
|
+- }
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
+- }
|
|||
|
|
+- });
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
+- }
|
|||
|
|
+- return future;
|
|||
|
|
++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
|
|||
|
|
++ CompletableFuture<SendResult> future0 = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ future0.complete(this.processSendResponse(brokerName, msg, response, brokerAddr));
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ future0.completeExceptionally(e);
|
|||
|
|
++ }
|
|||
|
|
++ return future0;
|
|||
|
|
++ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CompletableFuture<SendResult> sendMessageAsync(
|
|||
|
|
+@@ -216,17 +185,14 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ msgBatch.setBody(body);
|
|||
|
|
+
|
|||
|
|
+ request.setBody(body);
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- try {
|
|||
|
|
+- future.complete(this.processSendResponse(brokerName, msgBatch, response, brokerAddr));
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- future.completeExceptionally(e);
|
|||
|
|
+- }
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
|
|||
|
|
++ CompletableFuture<SendResult> future0 = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ future0.complete(processSendResponse(brokerName, msgBatch, response, brokerAddr));
|
|||
|
|
++ } catch (Exception e) {
|
|||
|
|
++ future0.completeExceptionally(e);
|
|||
|
|
+ }
|
|||
|
|
++ return future0;
|
|||
|
|
+ });
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+ future.completeExceptionally(t);
|
|||
|
|
+@@ -240,21 +206,7 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ long timeoutMillis
|
|||
|
|
+ ) {
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader);
|
|||
|
|
+-
|
|||
|
|
+- CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
+- try {
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- future.complete(response);
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
+- }
|
|||
|
|
+- });
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
+- }
|
|||
|
|
+- return future;
|
|||
|
|
++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CompletableFuture<PopResult> popMessageAsync(
|
|||
|
|
+@@ -402,38 +354,31 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ QueryConsumerOffsetRequestHeader requestHeader,
|
|||
|
|
+ long timeoutMillis
|
|||
|
|
+ ) {
|
|||
|
|
+- CompletableFuture<Long> future = new CompletableFuture<>();
|
|||
|
|
+- try {
|
|||
|
|
+- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader);
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- switch (response.getCode()) {
|
|||
|
|
+- case ResponseCode.SUCCESS: {
|
|||
|
|
+- try {
|
|||
|
|
+- QueryConsumerOffsetResponseHeader responseHeader =
|
|||
|
|
+- (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class);
|
|||
|
|
+- future.complete(responseHeader.getOffset());
|
|||
|
|
+- } catch (RemotingCommandException e) {
|
|||
|
|
+- future.completeExceptionally(e);
|
|||
|
|
+- }
|
|||
|
|
+- break;
|
|||
|
|
+- }
|
|||
|
|
+- case ResponseCode.QUERY_NOT_FOUND: {
|
|||
|
|
+- future.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr));
|
|||
|
|
+- break;
|
|||
|
|
+- }
|
|||
|
|
+- default:
|
|||
|
|
+- break;
|
|||
|
|
++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader);
|
|||
|
|
++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
|
|||
|
|
++ CompletableFuture<Long> future0 = new CompletableFuture<>();
|
|||
|
|
++ switch (response.getCode()) {
|
|||
|
|
++ case ResponseCode.SUCCESS: {
|
|||
|
|
++ try {
|
|||
|
|
++ QueryConsumerOffsetResponseHeader responseHeader =
|
|||
|
|
++ (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class);
|
|||
|
|
++ future0.complete(responseHeader.getOffset());
|
|||
|
|
++ } catch (RemotingCommandException e) {
|
|||
|
|
++ future0.completeExceptionally(e);
|
|||
|
|
+ }
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
++ break;
|
|||
|
|
+ }
|
|||
|
|
+- });
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
+- }
|
|||
|
|
+- return future;
|
|||
|
|
++ case ResponseCode.QUERY_NOT_FOUND: {
|
|||
|
|
++ future0.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr));
|
|||
|
|
++ break;
|
|||
|
|
++ }
|
|||
|
|
++ default: {
|
|||
|
|
++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
++ break;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ return future0;
|
|||
|
|
++ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CompletableFuture<Void> updateConsumerOffsetOneWay(
|
|||
|
|
+@@ -461,9 +406,14 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+
|
|||
|
|
+ CompletableFuture<List<String>> future = new CompletableFuture<>();
|
|||
|
|
+ try {
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ switch (response.getCode()) {
|
|||
|
|
+ case ResponseCode.SUCCESS: {
|
|||
|
|
+ if (response.getBody() != null) {
|
|||
|
|
+@@ -485,8 +435,11 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ break;
|
|||
|
|
+ }
|
|||
|
|
+ future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ future.completeExceptionally(throwable);
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+@@ -501,9 +454,14 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+
|
|||
|
|
+ CompletableFuture<Long> future = new CompletableFuture<>();
|
|||
|
|
+ try {
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ if (ResponseCode.SUCCESS == response.getCode()) {
|
|||
|
|
+ try {
|
|||
|
|
+ GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
|
|||
|
|
+@@ -513,8 +471,11 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ future.completeExceptionally(throwable);
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+@@ -529,9 +490,14 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+
|
|||
|
|
+ CompletableFuture<Long> future = new CompletableFuture<>();
|
|||
|
|
+ try {
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ if (ResponseCode.SUCCESS == response.getCode()) {
|
|||
|
|
+ try {
|
|||
|
|
+ GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
|
|||
|
|
+@@ -541,8 +507,11 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ future.completeExceptionally(throwable);
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+@@ -555,57 +524,41 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+ long timeoutMillis) {
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader);
|
|||
|
|
+
|
|||
|
|
+- CompletableFuture<Long> future = new CompletableFuture<>();
|
|||
|
|
+- try {
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
+- try {
|
|||
|
|
+- SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class);
|
|||
|
|
+- future.complete(responseHeader.getOffset());
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
|
|||
|
|
++ CompletableFuture<Long> future0 = new CompletableFuture<>();
|
|||
|
|
++ if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
++ try {
|
|||
|
|
++ SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class);
|
|||
|
|
++ future0.complete(responseHeader.getOffset());
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future0.completeExceptionally(t);
|
|||
|
|
+ }
|
|||
|
|
+- });
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
+- }
|
|||
|
|
+- return future;
|
|||
|
|
++ } else {
|
|||
|
|
++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
++ }
|
|||
|
|
++ return future0;
|
|||
|
|
++ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CompletableFuture<Set<MessageQueue>> lockBatchMQWithFuture(String brokerAddr,
|
|||
|
|
+ LockBatchRequestBody requestBody, long timeoutMillis) {
|
|||
|
|
+- CompletableFuture<Set<MessageQueue>> future = new CompletableFuture<>();
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null);
|
|||
|
|
+ request.setBody(requestBody.encode());
|
|||
|
|
+- try {
|
|||
|
|
+- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
+- if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
+- try {
|
|||
|
|
+- LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class);
|
|||
|
|
+- Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
|
|||
|
|
+- future.complete(messageQueues);
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
+- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(processNullResponseErr(responseFuture));
|
|||
|
|
++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
|
|||
|
|
++ CompletableFuture<Set<MessageQueue>> future0 = new CompletableFuture<>();
|
|||
|
|
++ if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
++ try {
|
|||
|
|
++ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class);
|
|||
|
|
++ Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
|
|||
|
|
++ future0.complete(messageQueues);
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future0.completeExceptionally(t);
|
|||
|
|
+ }
|
|||
|
|
+- });
|
|||
|
|
+- } catch (Exception e) {
|
|||
|
|
+- future.completeExceptionally(e);
|
|||
|
|
+- }
|
|||
|
|
+- return future;
|
|||
|
|
++ } else {
|
|||
|
|
++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
++ }
|
|||
|
|
++ return future0;
|
|||
|
|
++ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CompletableFuture<Void> unlockBatchMQOneway(String brokerAddr,
|
|||
|
|
+@@ -624,25 +577,21 @@ public class MQClientAPIExt extends MQClientAPIImpl {
|
|||
|
|
+
|
|||
|
|
+ public CompletableFuture<Boolean> notification(String brokerAddr, NotificationRequestHeader requestHeader,
|
|||
|
|
+ long timeoutMillis) {
|
|||
|
|
+- CompletableFuture<Boolean> future = new CompletableFuture<>();
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFICATION, requestHeader);
|
|||
|
|
+- try {
|
|||
|
|
+- this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenAccept(response -> {
|
|||
|
|
+- if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
+- try {
|
|||
|
|
+- NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class);
|
|||
|
|
+- future.complete(responseHeader.isHasMsg());
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
+- }
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
|
|||
|
|
++ CompletableFuture<Boolean> future0 = new CompletableFuture<>();
|
|||
|
|
++ if (response.getCode() == ResponseCode.SUCCESS) {
|
|||
|
|
++ try {
|
|||
|
|
++ NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class);
|
|||
|
|
++ future0.complete(responseHeader.isHasMsg());
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future0.completeExceptionally(t);
|
|||
|
|
+ }
|
|||
|
|
+- });
|
|||
|
|
+- } catch (Throwable t) {
|
|||
|
|
+- future.completeExceptionally(t);
|
|||
|
|
+- }
|
|||
|
|
+- return future;
|
|||
|
|
++ } else {
|
|||
|
|
++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
|
|||
|
|
++ }
|
|||
|
|
++ return future0;
|
|||
|
|
++ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CompletableFuture<RemotingCommand> invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) {
|
|||
|
|
+diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
|
|||
|
|
+index d13f2cfe4..c152d38ea 100644
|
|||
|
|
+--- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
|
|||
|
|
++++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
|
|||
|
|
+@@ -212,7 +212,7 @@ public class MQClientAPIImplTest {
|
|||
|
|
+ RemotingCommand request = mock.getArgument(1);
|
|||
|
|
+ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
|
|||
|
|
+ responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
+@@ -386,7 +386,7 @@ public class MQClientAPIImplTest {
|
|||
|
|
+ RemotingCommand request = mock.getArgument(1);
|
|||
|
|
+ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
|
|||
|
|
+ responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class));
|
|||
|
|
+@@ -472,7 +472,7 @@ public class MQClientAPIImplTest {
|
|||
|
|
+ message.putUserProperty("key", "value");
|
|||
|
|
+ response.setBody(MessageDecoder.encode(message, false));
|
|||
|
|
+ responseFuture.setResponseCommand(response);
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
+@@ -543,7 +543,7 @@ public class MQClientAPIImplTest {
|
|||
|
|
+ message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0));
|
|||
|
|
+ response.setBody(MessageDecoder.encode(message, false));
|
|||
|
|
+ responseFuture.setResponseCommand(response);
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
+@@ -585,7 +585,7 @@ public class MQClientAPIImplTest {
|
|||
|
|
+ response.setOpaque(request.getOpaque());
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+ responseFuture.setResponseCommand(response);
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
+@@ -622,7 +622,7 @@ public class MQClientAPIImplTest {
|
|||
|
|
+ responseHeader.setPopTime(System.currentTimeMillis());
|
|||
|
|
+ responseHeader.setInvisibleTime(10 * 1000L);
|
|||
|
|
+ responseFuture.setResponseCommand(response);
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
|
|||
|
|
+index fe07090d5..3227d1e1c 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
|
|||
|
|
+@@ -26,7 +26,6 @@ import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+ import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.rocketmq.acl.AccessValidator;
|
|||
|
|
+-import org.apache.rocketmq.client.exception.MQClientException;
|
|||
|
|
+ import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+ import org.apache.rocketmq.common.future.FutureTaskExt;
|
|||
|
|
+ import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
|
|||
|
|
+@@ -51,10 +50,12 @@ import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager;
|
|||
|
|
+ import org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline;
|
|||
|
|
+ import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
|
|||
|
|
+ import org.apache.rocketmq.remoting.ChannelEventListener;
|
|||
|
|
++import org.apache.rocketmq.remoting.InvokeCallback;
|
|||
|
|
+ import org.apache.rocketmq.remoting.RemotingServer;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.RequestTask;
|
|||
|
|
++import org.apache.rocketmq.remoting.netty.ResponseFuture;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RequestCode;
|
|||
|
|
+@@ -239,12 +240,21 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
|
|||
|
|
+ long timeoutMillis) {
|
|||
|
|
+ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
+ try {
|
|||
|
|
+- this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- if (responseFuture.getResponseCommand() == null) {
|
|||
|
|
+- future.completeExceptionally(new MQClientException("response is null after send request to client", responseFuture.getCause()));
|
|||
|
|
+- return;
|
|||
|
|
++ this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ future.complete(response);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ future.completeExceptionally(throwable);
|
|||
|
|
+ }
|
|||
|
|
+- future.complete(responseFuture.getResponseCommand());
|
|||
|
|
+ });
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+ future.completeExceptionally(t);
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
|
|||
|
|
+index 3f3a4ae40..e2d05b0f5 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
|
|||
|
|
+@@ -24,6 +24,7 @@ import java.util.ArrayList;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.UUID;
|
|||
|
|
++import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ThreadLocalRandom;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicReference;
|
|||
|
|
+ import java.util.stream.Collectors;
|
|||
|
|
+@@ -85,6 +86,7 @@ import static org.mockito.ArgumentMatchers.any;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyLong;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyString;
|
|||
|
|
+ import static org.mockito.Mockito.doAnswer;
|
|||
|
|
++import static org.mockito.Mockito.doReturn;
|
|||
|
|
+
|
|||
|
|
+ @RunWith(MockitoJUnitRunner.class)
|
|||
|
|
+ public class MQClientAPIExtTest {
|
|||
|
|
+@@ -109,13 +111,9 @@ public class MQClientAPIExtTest {
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testSendHeartbeatAsync() throws Exception {
|
|||
|
|
+- doAnswer((Answer<Void>) mock -> {
|
|||
|
|
+- InvokeCallback invokeCallback = mock.getArgument(3);
|
|||
|
|
+- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
|
|||
|
|
+- responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""));
|
|||
|
|
+- invokeCallback.operationComplete(responseFuture);
|
|||
|
|
+- return null;
|
|||
|
|
+- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
|
|||
|
|
++ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
++ future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""));
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
|
|||
|
|
+
|
|||
|
|
+ assertNotNull(mqClientAPI.sendHeartbeatAsync(BROKER_ADDR, new HeartbeatData(), TIMEOUT).get());
|
|||
|
|
+ }
|
|||
|
|
+@@ -123,20 +121,16 @@ public class MQClientAPIExtTest {
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testSendMessageAsync() throws Exception {
|
|||
|
|
+ AtomicReference<String> msgIdRef = new AtomicReference<>();
|
|||
|
|
+- doAnswer((Answer<Void>) mock -> {
|
|||
|
|
+- InvokeCallback invokeCallback = mock.getArgument(3);
|
|||
|
|
+- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
|
|||
|
|
+- RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
|
|||
|
|
+- SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();
|
|||
|
|
+- sendMessageResponseHeader.setMsgId(msgIdRef.get());
|
|||
|
|
+- sendMessageResponseHeader.setQueueId(0);
|
|||
|
|
+- sendMessageResponseHeader.setQueueOffset(1L);
|
|||
|
|
+- response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+- response.makeCustomHeaderToNet();
|
|||
|
|
+- responseFuture.putResponse(response);
|
|||
|
|
+- invokeCallback.operationComplete(responseFuture);
|
|||
|
|
+- return null;
|
|||
|
|
+- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
|
|||
|
|
++ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
++ RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
|
|||
|
|
++ SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();
|
|||
|
|
++ sendMessageResponseHeader.setMsgId(msgIdRef.get());
|
|||
|
|
++ sendMessageResponseHeader.setQueueId(0);
|
|||
|
|
++ sendMessageResponseHeader.setQueueOffset(1L);
|
|||
|
|
++ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
++ response.makeCustomHeaderToNet();
|
|||
|
|
++ future.complete(response);
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
|
|||
|
|
+
|
|||
|
|
+ MessageExt messageExt = createMessage();
|
|||
|
|
+ msgIdRef.set(MessageClientIDSetter.getUniqID(messageExt));
|
|||
|
|
+@@ -150,20 +144,16 @@ public class MQClientAPIExtTest {
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testSendMessageListAsync() throws Exception {
|
|||
|
|
+- doAnswer((Answer<Void>) mock -> {
|
|||
|
|
+- InvokeCallback invokeCallback = mock.getArgument(3);
|
|||
|
|
+- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
|
|||
|
|
+- RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
|
|||
|
|
+- SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();
|
|||
|
|
+- sendMessageResponseHeader.setMsgId("");
|
|||
|
|
+- sendMessageResponseHeader.setQueueId(0);
|
|||
|
|
+- sendMessageResponseHeader.setQueueOffset(1L);
|
|||
|
|
+- response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+- response.makeCustomHeaderToNet();
|
|||
|
|
+- responseFuture.putResponse(response);
|
|||
|
|
+- invokeCallback.operationComplete(responseFuture);
|
|||
|
|
+- return null;
|
|||
|
|
+- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
|
|||
|
|
++ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
++ RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
|
|||
|
|
++ SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();
|
|||
|
|
++ sendMessageResponseHeader.setMsgId("");
|
|||
|
|
++ sendMessageResponseHeader.setQueueId(0);
|
|||
|
|
++ sendMessageResponseHeader.setQueueOffset(1L);
|
|||
|
|
++ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
++ response.makeCustomHeaderToNet();
|
|||
|
|
++ future.complete(response);
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
|
|||
|
|
+
|
|||
|
|
+ List<MessageExt> messageExtList = new ArrayList<>();
|
|||
|
|
+ StringBuilder sb = new StringBuilder();
|
|||
|
|
+@@ -182,13 +172,9 @@ public class MQClientAPIExtTest {
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testSendMessageBackAsync() throws Exception {
|
|||
|
|
+- doAnswer((Answer<Void>) mock -> {
|
|||
|
|
+- InvokeCallback invokeCallback = mock.getArgument(3);
|
|||
|
|
+- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
|
|||
|
|
+- responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""));
|
|||
|
|
+- invokeCallback.operationComplete(responseFuture);
|
|||
|
|
+- return null;
|
|||
|
|
+- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
|
|||
|
|
++ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
++ future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""));
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
|
|||
|
|
+
|
|||
|
|
+ RemotingCommand remotingCommand = mqClientAPI.sendMessageBackAsync(BROKER_ADDR, new ConsumerSendMsgBackRequestHeader(), TIMEOUT)
|
|||
|
|
+ .get();
|
|||
|
|
+@@ -285,7 +271,7 @@ public class MQClientAPIExtTest {
|
|||
|
|
+ body.setConsumerIdList(clientIds);
|
|||
|
|
+ response.setBody(body.encode());
|
|||
|
|
+ responseFuture.putResponse(response);
|
|||
|
|
+- invokeCallback.operationComplete(responseFuture);
|
|||
|
|
++ invokeCallback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
|
|||
|
|
+
|
|||
|
|
+@@ -302,7 +288,7 @@ public class MQClientAPIExtTest {
|
|||
|
|
+ response.setCode(ResponseCode.SYSTEM_ERROR);
|
|||
|
|
+ response.makeCustomHeaderToNet();
|
|||
|
|
+ responseFuture.putResponse(response);
|
|||
|
|
+- invokeCallback.operationComplete(responseFuture);
|
|||
|
|
++ invokeCallback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
|
|||
|
|
+
|
|||
|
|
+@@ -322,7 +308,7 @@ public class MQClientAPIExtTest {
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+ response.makeCustomHeaderToNet();
|
|||
|
|
+ responseFuture.putResponse(response);
|
|||
|
|
+- invokeCallback.operationComplete(responseFuture);
|
|||
|
|
++ invokeCallback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
|
|||
|
|
+
|
|||
|
|
+@@ -335,18 +321,15 @@ public class MQClientAPIExtTest {
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testSearchOffsetAsync() throws Exception {
|
|||
|
|
+ long offset = ThreadLocalRandom.current().nextLong();
|
|||
|
|
+- doAnswer((Answer<Void>) mock -> {
|
|||
|
|
+- InvokeCallback invokeCallback = mock.getArgument(3);
|
|||
|
|
+- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
|
|||
|
|
+- RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);
|
|||
|
|
+- SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();
|
|||
|
|
+- responseHeader.setOffset(offset);
|
|||
|
|
+- response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+- response.makeCustomHeaderToNet();
|
|||
|
|
+- responseFuture.putResponse(response);
|
|||
|
|
+- invokeCallback.operationComplete(responseFuture);
|
|||
|
|
+- return null;
|
|||
|
|
+- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
|
|||
|
|
++ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
++ RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);
|
|||
|
|
++ SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();
|
|||
|
|
++ responseHeader.setOffset(offset);
|
|||
|
|
++ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
++ response.makeCustomHeaderToNet();
|
|||
|
|
++ future.complete(response);
|
|||
|
|
++
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
|
|||
|
|
+
|
|||
|
|
+ SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader();
|
|||
|
|
+ requestHeader.setTopic(TOPIC);
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java
|
|||
|
|
+index ce78fa923..6be491745 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java
|
|||
|
|
+@@ -17,7 +17,22 @@
|
|||
|
|
+ package org.apache.rocketmq.remoting;
|
|||
|
|
+
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.ResponseFuture;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+
|
|||
|
|
+ public interface InvokeCallback {
|
|||
|
|
++ /**
|
|||
|
|
++ * This method is expected to be invoked after {@link #operationSucceed(RemotingCommand)}
|
|||
|
|
++ * or {@link #operationFail(Throwable)}
|
|||
|
|
++ *
|
|||
|
|
++ * @param responseFuture the returned object contains response or exception
|
|||
|
|
++ */
|
|||
|
|
+ void operationComplete(final ResponseFuture responseFuture);
|
|||
|
|
++
|
|||
|
|
++ default void operationSucceed(final RemotingCommand response) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ default void operationFail(final Throwable throwable) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java
|
|||
|
|
+index ff0b3df95..c8389eedb 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java
|
|||
|
|
+@@ -20,11 +20,11 @@ import java.util.List;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingConnectException;
|
|||
|
|
+-import org.apache.rocketmq.remoting.exception.RemotingException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
|
|||
|
|
++import org.apache.rocketmq.remoting.netty.ResponseFuture;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+
|
|||
|
|
+ public interface RemotingClient extends RemotingService {
|
|||
|
|
+@@ -51,18 +51,21 @@ public interface RemotingClient extends RemotingService {
|
|||
|
|
+ final long timeoutMillis) {
|
|||
|
|
+ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
+ try {
|
|||
|
|
+- invokeAsync(addr, request, timeoutMillis, responseFuture -> {
|
|||
|
|
+- RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
+- if (response != null) {
|
|||
|
|
++ invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ future.complete(response);
|
|||
|
|
+- } else {
|
|||
|
|
+- if (!responseFuture.isSendRequestOK()) {
|
|||
|
|
+- future.completeExceptionally(new RemotingSendRequestException(addr, responseFuture.getCause()));
|
|||
|
|
+- } else if (responseFuture.isTimeout()) {
|
|||
|
|
+- future.completeExceptionally(new RemotingTimeoutException(addr, timeoutMillis, responseFuture.getCause()));
|
|||
|
|
+- } else {
|
|||
|
|
+- future.completeExceptionally(new RemotingException(request.toString(), responseFuture.getCause()));
|
|||
|
|
+- }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ future.completeExceptionally(throwable);
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+ } catch (Throwable t) {
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
|
|||
|
|
+index fce2de267..12e66f913 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
|
|||
|
|
+@@ -23,20 +23,23 @@ import io.netty.handler.ssl.SslContext;
|
|||
|
|
+ import io.netty.handler.ssl.SslHandler;
|
|||
|
|
+ import io.netty.util.concurrent.Future;
|
|||
|
|
+ import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
+-import java.net.SocketAddress;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+ import java.util.Iterator;
|
|||
|
|
+ import java.util.LinkedList;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Map.Entry;
|
|||
|
|
++import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
++import java.util.concurrent.ExecutionException;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
+ import java.util.concurrent.RejectedExecutionException;
|
|||
|
|
+ import java.util.concurrent.Semaphore;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
++import java.util.concurrent.TimeoutException;
|
|||
|
|
++import java.util.concurrent.atomic.AtomicReference;
|
|||
|
|
+ import java.util.function.Consumer;
|
|||
|
|
+ import javax.annotation.Nullable;
|
|||
|
|
+ import org.apache.rocketmq.common.AbortProcessException;
|
|||
|
|
+@@ -125,7 +128,7 @@ public abstract class NettyRemotingAbstract {
|
|||
|
|
+ * Constructor, specifying capacity of one-way and asynchronous semaphores.
|
|||
|
|
+ *
|
|||
|
|
+ * @param permitsOneway Number of permits for one-way requests.
|
|||
|
|
+- * @param permitsAsync Number of permits for asynchronous requests.
|
|||
|
|
++ * @param permitsAsync Number of permits for asynchronous requests.
|
|||
|
|
+ */
|
|||
|
|
+ public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) {
|
|||
|
|
+ this.semaphoreOneway = new Semaphore(permitsOneway, true);
|
|||
|
|
+@@ -367,8 +370,7 @@ public abstract class NettyRemotingAbstract {
|
|||
|
|
+ responseFuture.release();
|
|||
|
|
+ }
|
|||
|
|
+ } else {
|
|||
|
|
+- log.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
|
|||
|
|
+- log.warn(cmd.toString());
|
|||
|
|
++ log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -467,57 +469,68 @@ public abstract class NettyRemotingAbstract {
|
|||
|
|
+ public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request,
|
|||
|
|
+ final long timeoutMillis)
|
|||
|
|
+ throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException {
|
|||
|
|
+- //get the request id
|
|||
|
|
+- final int opaque = request.getOpaque();
|
|||
|
|
+-
|
|||
|
|
+ try {
|
|||
|
|
+- final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, null, null);
|
|||
|
|
+- this.responseTable.put(opaque, responseFuture);
|
|||
|
|
+- final SocketAddress addr = channel.remoteAddress();
|
|||
|
|
+- channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> {
|
|||
|
|
+- if (f.isSuccess()) {
|
|||
|
|
+- responseFuture.setSendRequestOK(true);
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+-
|
|||
|
|
+- responseFuture.setSendRequestOK(false);
|
|||
|
|
+- responseTable.remove(opaque);
|
|||
|
|
+- responseFuture.setCause(f.cause());
|
|||
|
|
+- responseFuture.putResponse(null);
|
|||
|
|
+- log.warn("Failed to write a request command to {}, caused by underlying I/O operation failure", addr);
|
|||
|
|
+- });
|
|||
|
|
++ return invokeImpl(channel, request, timeoutMillis).thenApply(ResponseFuture::getResponseCommand)
|
|||
|
|
++ .get(timeoutMillis, TimeUnit.MILLISECONDS);
|
|||
|
|
++ } catch (ExecutionException e) {
|
|||
|
|
++ throw new RemotingSendRequestException(channel.remoteAddress().toString(), e.getCause());
|
|||
|
|
++ } catch (TimeoutException e) {
|
|||
|
|
++ throw new RemotingTimeoutException(channel.remoteAddress().toString(), timeoutMillis, e.getCause());
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
+
|
|||
|
|
+- RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis);
|
|||
|
|
+- if (null == responseCommand) {
|
|||
|
|
+- if (responseFuture.isSendRequestOK()) {
|
|||
|
|
+- throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis,
|
|||
|
|
+- responseFuture.getCause());
|
|||
|
|
+- } else {
|
|||
|
|
+- throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause());
|
|||
|
|
+- }
|
|||
|
|
++ public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request,
|
|||
|
|
++ final long timeoutMillis) {
|
|||
|
|
++ String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
|
|||
|
|
++ doBeforeRpcHooks(channelRemoteAddr, request);
|
|||
|
|
++ return invoke0(channel, request, timeoutMillis).whenComplete((v, t) -> {
|
|||
|
|
++ if (t == null) {
|
|||
|
|
++ doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand());
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
+- return responseCommand;
|
|||
|
|
+- } finally {
|
|||
|
|
+- this.responseTable.remove(opaque);
|
|||
|
|
+- }
|
|||
|
|
++ });
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis,
|
|||
|
|
+- final InvokeCallback invokeCallback)
|
|||
|
|
+- throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {
|
|||
|
|
++ protected CompletableFuture<ResponseFuture> invoke0(final Channel channel, final RemotingCommand request,
|
|||
|
|
++ final long timeoutMillis) {
|
|||
|
|
++ CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
|
|||
|
|
+ long beginStartTime = System.currentTimeMillis();
|
|||
|
|
+ final int opaque = request.getOpaque();
|
|||
|
|
+- boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
|
|||
|
|
++
|
|||
|
|
++ boolean acquired;
|
|||
|
|
++ try {
|
|||
|
|
++ acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ return future;
|
|||
|
|
++ }
|
|||
|
|
+ if (acquired) {
|
|||
|
|
+ final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync);
|
|||
|
|
+ long costTime = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
+ if (timeoutMillis < costTime) {
|
|||
|
|
+ once.release();
|
|||
|
|
+- throw new RemotingTimeoutException("invokeAsyncImpl call timeout");
|
|||
|
|
++ future.completeExceptionally(new RemotingTimeoutException("invokeAsyncImpl call timeout"));
|
|||
|
|
++ return future;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once);
|
|||
|
|
++ AtomicReference<ResponseFuture> responseFutureReference = new AtomicReference<>();
|
|||
|
|
++ final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, request, timeoutMillis - costTime,
|
|||
|
|
++ new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ future.complete(responseFutureReference.get());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ future.completeExceptionally(throwable);
|
|||
|
|
++ }
|
|||
|
|
++ }, once);
|
|||
|
|
++ responseFutureReference.set(responseFuture);
|
|||
|
|
+ this.responseTable.put(opaque, responseFuture);
|
|||
|
|
+ try {
|
|||
|
|
+ channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> {
|
|||
|
|
+@@ -528,15 +541,17 @@ public abstract class NettyRemotingAbstract {
|
|||
|
|
+ requestFail(opaque);
|
|||
|
|
+ log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel));
|
|||
|
|
+ });
|
|||
|
|
++ return future;
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ responseTable.remove(opaque);
|
|||
|
|
+ responseFuture.release();
|
|||
|
|
+ log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e);
|
|||
|
|
+- throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e);
|
|||
|
|
++ future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e));
|
|||
|
|
++ return future;
|
|||
|
|
+ }
|
|||
|
|
+ } else {
|
|||
|
|
+ if (timeoutMillis <= 0) {
|
|||
|
|
+- throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast");
|
|||
|
|
++ future.completeExceptionally(new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast"));
|
|||
|
|
+ } else {
|
|||
|
|
+ String info =
|
|||
|
|
+ String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d",
|
|||
|
|
+@@ -545,11 +560,31 @@ public abstract class NettyRemotingAbstract {
|
|||
|
|
+ this.semaphoreAsync.availablePermits()
|
|||
|
|
+ );
|
|||
|
|
+ log.warn(info);
|
|||
|
|
+- throw new RemotingTimeoutException(info);
|
|||
|
|
++ future.completeExceptionally(new RemotingTimeoutException(info));
|
|||
|
|
+ }
|
|||
|
|
++ return future;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis,
|
|||
|
|
++ final InvokeCallback invokeCallback) {
|
|||
|
|
++ invokeImpl(channel, request, timeoutMillis)
|
|||
|
|
++ .whenComplete((v, t) -> {
|
|||
|
|
++ if (t == null) {
|
|||
|
|
++ invokeCallback.operationComplete(v);
|
|||
|
|
++ } else {
|
|||
|
|
++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, timeoutMillis, null, null);
|
|||
|
|
++ responseFuture.setCause(t);
|
|||
|
|
++ invokeCallback.operationComplete(responseFuture);
|
|||
|
|
++ }
|
|||
|
|
++ })
|
|||
|
|
++ .thenAccept(responseFuture -> invokeCallback.operationSucceed(responseFuture.getResponseCommand()))
|
|||
|
|
++ .exceptionally(t -> {
|
|||
|
|
++ invokeCallback.operationFail(t);
|
|||
|
|
++ return null;
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ private void requestFail(final int opaque) {
|
|||
|
|
+ ResponseFuture responseFuture = responseTable.remove(opaque);
|
|||
|
|
+ if (responseFuture != null) {
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
+index 64621dd6c..d784351a5 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
+@@ -527,15 +527,13 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ if (channel != null && channel.isActive()) {
|
|||
|
|
+ long left = timeoutMillis;
|
|||
|
|
+ try {
|
|||
|
|
+- doBeforeRpcHooks(channelRemoteAddr, request);
|
|||
|
|
+ long costTime = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
+ left -= costTime;
|
|||
|
|
+ if (left <= 0) {
|
|||
|
|
+ throw new RemotingTimeoutException("invokeSync call the addr[" + channelRemoteAddr + "] timeout");
|
|||
|
|
+ }
|
|||
|
|
+ RemotingCommand response = this.invokeSyncImpl(channel, request, left);
|
|||
|
|
+- doAfterRpcHooks(channelRemoteAddr, request, response);
|
|||
|
|
+- this.updateChannelLastResponseTime(addr);
|
|||
|
|
++ updateChannelLastResponseTime(addr);
|
|||
|
|
+ return response;
|
|||
|
|
+ } catch (RemotingSendRequestException e) {
|
|||
|
|
+ LOGGER.warn("invokeSync: send request exception, so close the channel[{}]", channelRemoteAddr);
|
|||
|
|
+@@ -727,18 +725,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ final Channel channel = this.getAndCreateChannel(addr);
|
|||
|
|
+ String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
|
|||
|
|
+ if (channel != null && channel.isActive()) {
|
|||
|
|
+- try {
|
|||
|
|
+- doBeforeRpcHooks(channelRemoteAddr, request);
|
|||
|
|
+- long costTime = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
+- if (timeoutMillis < costTime) {
|
|||
|
|
+- throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout");
|
|||
|
|
+- }
|
|||
|
|
+- this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr));
|
|||
|
|
+- } catch (RemotingSendRequestException e) {
|
|||
|
|
+- LOGGER.warn("invokeAsync: send request exception, so close the channel[{}]", channelRemoteAddr);
|
|||
|
|
+- this.closeChannel(addr, channel);
|
|||
|
|
+- throw e;
|
|||
|
|
++ long costTime = System.currentTimeMillis() - beginStartTime;
|
|||
|
|
++ if (timeoutMillis < costTime) {
|
|||
|
|
++ throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout");
|
|||
|
|
+ }
|
|||
|
|
++ this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr));
|
|||
|
|
+ } else {
|
|||
|
|
+ this.closeChannel(addr, channel);
|
|||
|
|
+ throw new RemotingConnectException(addr);
|
|||
|
|
+@@ -931,11 +922,19 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
+- if (responseFuture != null && responseFuture.isSendRequestOK() && responseFuture.getResponseCommand() != null) {
|
|||
|
|
+- NettyRemotingClient.this.updateChannelLastResponseTime(addr);
|
|||
|
|
+- }
|
|||
|
|
+ this.invokeCallback.operationComplete(responseFuture);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ updateChannelLastResponseTime(addr);
|
|||
|
|
++ this.invokeCallback.operationSucceed(response);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(final Throwable throwable) {
|
|||
|
|
++ this.invokeCallback.operationFail(throwable);
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ class NettyClientHandler extends SimpleChannelInboundHandler<RemotingCommand> {
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java
|
|||
|
|
+index 19f705d74..0882818fe 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java
|
|||
|
|
+@@ -22,6 +22,9 @@ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicBoolean;
|
|||
|
|
+ import org.apache.rocketmq.remoting.InvokeCallback;
|
|||
|
|
+ import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;
|
|||
|
|
++import org.apache.rocketmq.remoting.exception.RemotingException;
|
|||
|
|
++import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
|
|||
|
|
++import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+
|
|||
|
|
+ public class ResponseFuture {
|
|||
|
|
+@@ -59,6 +62,18 @@ public class ResponseFuture {
|
|||
|
|
+ public void executeInvokeCallback() {
|
|||
|
|
+ if (invokeCallback != null) {
|
|||
|
|
+ if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) {
|
|||
|
|
++ RemotingCommand response = getResponseCommand();
|
|||
|
|
++ if (response != null) {
|
|||
|
|
++ invokeCallback.operationSucceed(response);
|
|||
|
|
++ } else {
|
|||
|
|
++ if (!isSendRequestOK()) {
|
|||
|
|
++ invokeCallback.operationFail(new RemotingSendRequestException(channel.remoteAddress().toString(), getCause()));
|
|||
|
|
++ } else if (isTimeout()) {
|
|||
|
|
++ invokeCallback.operationFail(new RemotingTimeoutException(channel.remoteAddress().toString(), getTimeoutMillis(), getCause()));
|
|||
|
|
++ } else {
|
|||
|
|
++ invokeCallback.operationFail(new RemotingException(getRequestCommand().toString(), getCause()));
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
+ invokeCallback.operationComplete(this);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java
|
|||
|
|
+index 133e0ed31..5328e8845 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java
|
|||
|
|
+@@ -160,31 +160,38 @@ public class RpcClientImpl implements RpcClient {
|
|||
|
|
+ InvokeCallback callback = new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
+- RemotingCommand responseCommand = responseFuture.getResponseCommand();
|
|||
|
|
+- if (responseCommand == null) {
|
|||
|
|
+- processFailedResponse(addr, requestCommand, responseFuture, rpcResponsePromise);
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ try {
|
|||
|
|
+- switch (responseCommand.getCode()) {
|
|||
|
|
++ switch (response.getCode()) {
|
|||
|
|
+ case ResponseCode.SUCCESS:
|
|||
|
|
+ case ResponseCode.PULL_NOT_FOUND:
|
|||
|
|
+ case ResponseCode.PULL_RETRY_IMMEDIATELY:
|
|||
|
|
+ case ResponseCode.PULL_OFFSET_MOVED:
|
|||
|
|
+ PullMessageResponseHeader responseHeader =
|
|||
|
|
+- (PullMessageResponseHeader) responseCommand.decodeCommandCustomHeader(PullMessageResponseHeader.class);
|
|||
|
|
+- rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));
|
|||
|
|
++ (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class);
|
|||
|
|
++ rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody()));
|
|||
|
|
+ default:
|
|||
|
|
+- RpcResponse rpcResponse = new RpcResponse(new RpcException(responseCommand.getCode(), "unexpected remote response code"));
|
|||
|
|
++ RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), "unexpected remote response code"));
|
|||
|
|
+ rpcResponsePromise.setSuccess(rpcResponse);
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+- String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand;
|
|||
|
|
+- RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e));
|
|||
|
|
++ String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + requestCommand;
|
|||
|
|
++ RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e));
|
|||
|
|
+ rpcResponsePromise.setSuccess(rpcResponse);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++ String errorMessage = "process failed. addr: " + addr + ". Request: " + requestCommand;
|
|||
|
|
++ RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, throwable));
|
|||
|
|
++ rpcResponsePromise.setSuccess(rpcResponse);
|
|||
|
|
++ }
|
|||
|
|
+ };
|
|||
|
|
+
|
|||
|
|
+ this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback);
|
|||
|
|
+diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java
|
|||
|
|
+index 90072960b..d0da0eb2e 100644
|
|||
|
|
+--- a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java
|
|||
|
|
++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java
|
|||
|
|
+@@ -26,12 +26,12 @@ import org.apache.rocketmq.remoting.exception.RemotingConnectException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
|
|||
|
|
+-import org.apache.rocketmq.remoting.netty.ResponseFuture;
|
|||
|
|
+-import org.apache.rocketmq.remoting.netty.NettyServerConfig;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyClientConfig;
|
|||
|
|
+-import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
|
|||
|
|
++import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
|
|||
|
|
+ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
|
|||
|
|
++import org.apache.rocketmq.remoting.netty.NettyServerConfig;
|
|||
|
|
++import org.apache.rocketmq.remoting.netty.ResponseFuture;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.LanguageCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+ import org.junit.AfterClass;
|
|||
|
|
+@@ -40,7 +40,6 @@ import org.junit.Test;
|
|||
|
|
+
|
|||
|
|
+ import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
+ import static org.junit.Assert.assertNotNull;
|
|||
|
|
+-import static org.junit.Assert.assertTrue;
|
|||
|
|
+
|
|||
|
|
+ public class RemotingServerTest {
|
|||
|
|
+ private static RemotingServer remotingServer;
|
|||
|
|
+@@ -122,10 +121,19 @@ public class RemotingServerTest {
|
|||
|
|
+ remotingClient.invokeAsync("localhost:" + remotingServer.localListenPort(), request, 1000 * 3, new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ latch.countDown();
|
|||
|
|
+- assertTrue(responseFuture != null);
|
|||
|
|
+- assertThat(responseFuture.getResponseCommand().getLanguage()).isEqualTo(LanguageCode.JAVA);
|
|||
|
|
+- assertThat(responseFuture.getResponseCommand().getExtFields()).hasSize(2);
|
|||
|
|
++ assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA);
|
|||
|
|
++ assertThat(response.getExtFields()).hasSize(2);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++
|
|||
|
|
+ }
|
|||
|
|
+ });
|
|||
|
|
+ latch.await();
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java
|
|||
|
|
+similarity index 57%
|
|||
|
|
+rename from client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java
|
|||
|
|
+rename to remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java
|
|||
|
|
+index 80188832e..8ddcdf35d 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java
|
|||
|
|
++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java
|
|||
|
|
+@@ -15,23 +15,14 @@
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+
|
|||
|
|
+-package org.apache.rocketmq.client.impl;
|
|||
|
|
++package org.apache.rocketmq.remoting.netty;
|
|||
|
|
+
|
|||
|
|
+-import org.apache.rocketmq.remoting.InvokeCallback;
|
|||
|
|
+-import org.apache.rocketmq.remoting.netty.ResponseFuture;
|
|||
|
|
+-
|
|||
|
|
+-public abstract class BaseInvokeCallback implements InvokeCallback {
|
|||
|
|
+- private final MQClientAPIImpl mqClientAPI;
|
|||
|
|
+-
|
|||
|
|
+- public BaseInvokeCallback(MQClientAPIImpl mqClientAPI) {
|
|||
|
|
+- this.mqClientAPI = mqClientAPI;
|
|||
|
|
+- }
|
|||
|
|
++import io.netty.channel.ChannelFuture;
|
|||
|
|
++import io.netty.channel.local.LocalChannel;
|
|||
|
|
+
|
|||
|
|
++public class MockChannel extends LocalChannel {
|
|||
|
|
+ @Override
|
|||
|
|
+- public void operationComplete(final ResponseFuture responseFuture) {
|
|||
|
|
+- mqClientAPI.execRpcHooksAfterRequest(responseFuture);
|
|||
|
|
+- onComplete(responseFuture);
|
|||
|
|
++ public ChannelFuture writeAndFlush(Object msg) {
|
|||
|
|
++ return new MockChannelPromise(MockChannel.this);
|
|||
|
|
+ }
|
|||
|
|
+-
|
|||
|
|
+- public abstract void onComplete(final ResponseFuture responseFuture);
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java
|
|||
|
|
+new file mode 100644
|
|||
|
|
+index 000000000..9c3a35487
|
|||
|
|
+--- /dev/null
|
|||
|
|
++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java
|
|||
|
|
+@@ -0,0 +1,191 @@
|
|||
|
|
++/*
|
|||
|
|
++ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
++ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
++ * this work for additional information regarding copyright ownership.
|
|||
|
|
++ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
++ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
++ * the License. You may obtain a copy of the License at
|
|||
|
|
++ *
|
|||
|
|
++ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
++ *
|
|||
|
|
++ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
++ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
++ * See the License for the specific language governing permissions and
|
|||
|
|
++ * limitations under the License.
|
|||
|
|
++ */
|
|||
|
|
++
|
|||
|
|
++package org.apache.rocketmq.remoting.netty;
|
|||
|
|
++
|
|||
|
|
++import io.netty.channel.Channel;
|
|||
|
|
++import io.netty.channel.ChannelPromise;
|
|||
|
|
++import io.netty.util.concurrent.Future;
|
|||
|
|
++import io.netty.util.concurrent.GenericFutureListener;
|
|||
|
|
++import java.util.concurrent.ExecutionException;
|
|||
|
|
++import java.util.concurrent.TimeUnit;
|
|||
|
|
++import java.util.concurrent.TimeoutException;
|
|||
|
|
++import org.jetbrains.annotations.NotNull;
|
|||
|
|
++
|
|||
|
|
++public class MockChannelPromise implements ChannelPromise {
|
|||
|
|
++ protected Channel channel;
|
|||
|
|
++
|
|||
|
|
++ public MockChannelPromise(Channel channel) {
|
|||
|
|
++ this.channel = channel;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public Channel channel() {
|
|||
|
|
++ return channel;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise setSuccess(Void result) {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise setSuccess() {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean trySuccess() {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise setFailure(Throwable cause) {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise addListener(GenericFutureListener<? extends Future<? super Void>> listener) {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise addListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise removeListener(GenericFutureListener<? extends Future<? super Void>> listener) {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise removeListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise sync() throws InterruptedException {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise syncUninterruptibly() {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise await() throws InterruptedException {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise awaitUninterruptibly() {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelPromise unvoid() {
|
|||
|
|
++ return this;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean isVoid() {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean trySuccess(Void result) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean tryFailure(Throwable cause) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean setUncancellable() {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean isSuccess() {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean isCancellable() {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public Throwable cause() {
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean await(long timeoutMillis) throws InterruptedException {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean awaitUninterruptibly(long timeout, TimeUnit unit) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean awaitUninterruptibly(long timeoutMillis) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public Void getNow() {
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean cancel(boolean mayInterruptIfRunning) {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean isCancelled() {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public boolean isDone() {
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public Void get() throws InterruptedException, ExecutionException {
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public Void get(long timeout,
|
|||
|
|
++ @NotNull java.util.concurrent.TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
|
|||
|
|
++ return null;
|
|||
|
|
++ }
|
|||
|
|
++}
|
|||
|
|
+diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java
|
|||
|
|
+index 8381c132b..dbbea86ea 100644
|
|||
|
|
+--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java
|
|||
|
|
++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java
|
|||
|
|
+@@ -39,9 +39,19 @@ public class NettyRemotingAbstractTest {
|
|||
|
|
+ final Semaphore semaphore = new Semaphore(0);
|
|||
|
|
+ ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+- public void operationComplete(final ResponseFuture responseFuture) {
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ assertThat(semaphore.availablePermits()).isEqualTo(0);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
+ }, new SemaphoreReleaseOnlyOnce(semaphore));
|
|||
|
|
+
|
|||
|
|
+ remotingAbstract.responseTable.putIfAbsent(1, responseFuture);
|
|||
|
|
+@@ -75,9 +85,19 @@ public class NettyRemotingAbstractTest {
|
|||
|
|
+ final Semaphore semaphore = new Semaphore(0);
|
|||
|
|
+ ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+- public void operationComplete(final ResponseFuture responseFuture) {
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
+ assertThat(semaphore.availablePermits()).isEqualTo(0);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
+ }, new SemaphoreReleaseOnlyOnce(semaphore));
|
|||
|
|
+
|
|||
|
|
+ remotingAbstract.responseTable.putIfAbsent(1, responseFuture);
|
|||
|
|
+@@ -98,7 +118,18 @@ public class NettyRemotingAbstractTest {
|
|||
|
|
+ // mock timeout
|
|||
|
|
+ ResponseFuture responseFuture = new ResponseFuture(null, dummyId, -1000, new InvokeCallback() {
|
|||
|
|
+ @Override
|
|||
|
|
+- public void operationComplete(final ResponseFuture responseFuture) {
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++
|
|||
|
|
+ }
|
|||
|
|
+ }, null);
|
|||
|
|
+ remotingAbstract.responseTable.putIfAbsent(dummyId, responseFuture);
|
|||
|
|
+@@ -111,7 +142,22 @@ public class NettyRemotingAbstractTest {
|
|||
|
|
+ final Semaphore semaphore = new Semaphore(0);
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(1, null);
|
|||
|
|
+ ResponseFuture responseFuture = new ResponseFuture(null, 1, request, 3000,
|
|||
|
|
+- responseFuture1 -> assertThat(semaphore.availablePermits()).isEqualTo(0), new SemaphoreReleaseOnlyOnce(semaphore));
|
|||
|
|
++ new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationSucceed(RemotingCommand response) {
|
|||
|
|
++ assertThat(semaphore.availablePermits()).isEqualTo(0);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationFail(Throwable throwable) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++ }, new SemaphoreReleaseOnlyOnce(semaphore));
|
|||
|
|
+
|
|||
|
|
+ remotingAbstract.responseTable.putIfAbsent(1, responseFuture);
|
|||
|
|
+ RemotingCommand response = RemotingCommand.createResponseCommand(0, "Foo");
|
|||
|
|
+diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
|
|||
|
|
+index 8fabbb21d..e72e7bd53 100644
|
|||
|
|
+--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
|
|||
|
|
++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
|
|||
|
|
+@@ -16,10 +16,17 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.remoting.netty;
|
|||
|
|
+
|
|||
|
|
++import io.netty.channel.Channel;
|
|||
|
|
++import io.netty.channel.ChannelFuture;
|
|||
|
|
++import io.netty.channel.local.LocalChannel;
|
|||
|
|
+ import java.util.concurrent.CompletableFuture;
|
|||
|
|
++import java.util.concurrent.ExecutionException;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+ import java.util.concurrent.Executors;
|
|||
|
|
++import java.util.concurrent.Semaphore;
|
|||
|
|
+ import org.apache.rocketmq.remoting.InvokeCallback;
|
|||
|
|
++import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
++import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingConnectException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
|
|||
|
|
+@@ -29,23 +36,33 @@ import org.apache.rocketmq.remoting.protocol.RequestCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.ResponseCode;
|
|||
|
|
+ import org.junit.Test;
|
|||
|
|
+ import org.junit.runner.RunWith;
|
|||
|
|
++import org.mockito.Mock;
|
|||
|
|
+ import org.mockito.Spy;
|
|||
|
|
+ import org.mockito.junit.MockitoJUnitRunner;
|
|||
|
|
+
|
|||
|
|
+ import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
++import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
|||
|
|
+ import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.any;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyLong;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyString;
|
|||
|
|
++import static org.mockito.ArgumentMatchers.eq;
|
|||
|
|
+ import static org.mockito.Mockito.doAnswer;
|
|||
|
|
++import static org.mockito.Mockito.doReturn;
|
|||
|
|
++import static org.mockito.Mockito.mock;
|
|||
|
|
++import static org.mockito.Mockito.never;
|
|||
|
|
++import static org.mockito.Mockito.times;
|
|||
|
|
++import static org.mockito.Mockito.verify;
|
|||
|
|
+
|
|||
|
|
+ @RunWith(MockitoJUnitRunner.class)
|
|||
|
|
+ public class NettyRemotingClientTest {
|
|||
|
|
+ @Spy
|
|||
|
|
+ private NettyRemotingClient remotingClient = new NettyRemotingClient(new NettyClientConfig());
|
|||
|
|
++ @Mock
|
|||
|
|
++ private RPCHook rpcHookMock;
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+- public void testSetCallbackExecutor() throws NoSuchFieldException, IllegalAccessException {
|
|||
|
|
++ public void testSetCallbackExecutor() {
|
|||
|
|
+ ExecutorService customized = Executors.newCachedThreadPool();
|
|||
|
|
+ remotingClient.setCallbackExecutor(customized);
|
|||
|
|
+ assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized);
|
|||
|
|
+@@ -61,7 +78,7 @@ public class NettyRemotingClientTest {
|
|||
|
|
+ InvokeCallback callback = invocation.getArgument(3);
|
|||
|
|
+ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
|
|||
|
|
+ responseFuture.setResponseCommand(response);
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+ return null;
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
+
|
|||
|
|
+@@ -78,9 +95,7 @@ public class NettyRemotingClientTest {
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+ doAnswer(invocation -> {
|
|||
|
|
+ InvokeCallback callback = invocation.getArgument(3);
|
|||
|
|
+- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
|
|||
|
|
+- responseFuture.setSendRequestOK(false);
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationFail(new RemotingSendRequestException(null));
|
|||
|
|
+ return null;
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
+
|
|||
|
|
+@@ -97,8 +112,7 @@ public class NettyRemotingClientTest {
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+ doAnswer(invocation -> {
|
|||
|
|
+ InvokeCallback callback = invocation.getArgument(3);
|
|||
|
|
+- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), -1L, null, null);
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationFail(new RemotingTimeoutException(""));
|
|||
|
|
+ return null;
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
+
|
|||
|
|
+@@ -115,8 +129,7 @@ public class NettyRemotingClientTest {
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+ doAnswer(invocation -> {
|
|||
|
|
+ InvokeCallback callback = invocation.getArgument(3);
|
|||
|
|
+- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
|
|||
|
|
+- callback.operationComplete(responseFuture);
|
|||
|
|
++ callback.operationFail(new RemotingException(null));
|
|||
|
|
+ return null;
|
|||
|
|
+ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
+
|
|||
|
|
+@@ -134,4 +147,158 @@ public class NettyRemotingClientTest {
|
|||
|
|
+ assertThat(e.getMessage()).contains(addr);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testInvoke0() throws ExecutionException, InterruptedException {
|
|||
|
|
++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
|
|||
|
|
++ RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
++ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
++ Channel channel = new MockChannel() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelFuture writeAndFlush(Object msg) {
|
|||
|
|
++ ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque());
|
|||
|
|
++ responseFuture.setResponseCommand(response);
|
|||
|
|
++ responseFuture.executeInvokeCallback();
|
|||
|
|
++ return super.writeAndFlush(msg);
|
|||
|
|
++ }
|
|||
|
|
++ };
|
|||
|
|
++ CompletableFuture<ResponseFuture> future = remotingClient.invoke0(channel, request, 1000L);
|
|||
|
|
++ assertThat(future.get().getResponseCommand()).isEqualTo(response);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testInvoke0WithException() {
|
|||
|
|
++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
|
|||
|
|
++ RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
++ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
++ Channel channel = new MockChannel() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public ChannelFuture writeAndFlush(Object msg) {
|
|||
|
|
++ ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque());
|
|||
|
|
++ responseFuture.executeInvokeCallback();
|
|||
|
|
++ return super.writeAndFlush(msg);
|
|||
|
|
++ }
|
|||
|
|
++ };
|
|||
|
|
++ CompletableFuture<ResponseFuture> future = remotingClient.invoke0(channel, request, 1000L);
|
|||
|
|
++ assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingException.class);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testInvokeSync() throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException {
|
|||
|
|
++ remotingClient.registerRPCHook(rpcHookMock);
|
|||
|
|
++
|
|||
|
|
++ Channel channel = new LocalChannel();
|
|||
|
|
++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
|
|||
|
|
++ RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
++ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
|
|||
|
|
++ responseFuture.setResponseCommand(response);
|
|||
|
|
++ CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
|
|||
|
|
++ future.complete(responseFuture);
|
|||
|
|
++
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
|
|||
|
|
++ RemotingCommand actual = remotingClient.invokeSyncImpl(channel, request, 1000);
|
|||
|
|
++ assertThat(actual).isEqualTo(response);
|
|||
|
|
++
|
|||
|
|
++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
|
|||
|
|
++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testInvokeAsync() {
|
|||
|
|
++ remotingClient.registerRPCHook(rpcHookMock);
|
|||
|
|
++ Channel channel = new LocalChannel();
|
|||
|
|
++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
|
|||
|
|
++ RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
++ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
|
|||
|
|
++ responseFuture.setResponseCommand(response);
|
|||
|
|
++ CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
|
|||
|
|
++ future.complete(responseFuture);
|
|||
|
|
++
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
|
|||
|
|
++
|
|||
|
|
++ InvokeCallback callback = mock(InvokeCallback.class);
|
|||
|
|
++ remotingClient.invokeAsyncImpl(channel, request, 1000, callback);
|
|||
|
|
++ verify(callback, times(1)).operationSucceed(eq(response));
|
|||
|
|
++ verify(callback, times(1)).operationComplete(eq(responseFuture));
|
|||
|
|
++ verify(callback, never()).operationFail(any());
|
|||
|
|
++
|
|||
|
|
++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
|
|||
|
|
++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testInvokeAsyncFail() {
|
|||
|
|
++ remotingClient.registerRPCHook(rpcHookMock);
|
|||
|
|
++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
|
|||
|
|
++
|
|||
|
|
++ Channel channel = new LocalChannel();
|
|||
|
|
++ CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
|
|||
|
|
++ future.completeExceptionally(new RemotingException(null));
|
|||
|
|
++
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
|
|||
|
|
++
|
|||
|
|
++ InvokeCallback callback = mock(InvokeCallback.class);
|
|||
|
|
++ remotingClient.invokeAsyncImpl(channel, request, 1000, callback);
|
|||
|
|
++ verify(callback, never()).operationSucceed(any());
|
|||
|
|
++ verify(callback, times(1)).operationComplete(any());
|
|||
|
|
++ verify(callback, times(1)).operationFail(any());
|
|||
|
|
++
|
|||
|
|
++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
|
|||
|
|
++ verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testInvokeImpl() throws ExecutionException, InterruptedException {
|
|||
|
|
++ remotingClient.registerRPCHook(rpcHookMock);
|
|||
|
|
++ Channel channel = new LocalChannel();
|
|||
|
|
++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
|
|||
|
|
++ RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
++ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() {
|
|||
|
|
++ @Override
|
|||
|
|
++ public void operationComplete(ResponseFuture responseFuture) {
|
|||
|
|
++
|
|||
|
|
++ }
|
|||
|
|
++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
|
|||
|
|
++ responseFuture.setResponseCommand(response);
|
|||
|
|
++ CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
|
|||
|
|
++ future.complete(responseFuture);
|
|||
|
|
++
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
|
|||
|
|
++
|
|||
|
|
++ CompletableFuture<ResponseFuture> future0 = remotingClient.invokeImpl(channel, request, 1000);
|
|||
|
|
++ assertThat(future0.get()).isEqualTo(responseFuture);
|
|||
|
|
++
|
|||
|
|
++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
|
|||
|
|
++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response));
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testInvokeImplFail() {
|
|||
|
|
++ remotingClient.registerRPCHook(rpcHookMock);
|
|||
|
|
++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
|
|||
|
|
++
|
|||
|
|
++ Channel channel = new LocalChannel();
|
|||
|
|
++ CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
|
|||
|
|
++ future.completeExceptionally(new RemotingException(null));
|
|||
|
|
++
|
|||
|
|
++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
|
|||
|
|
++
|
|||
|
|
++ assertThatThrownBy(() -> remotingClient.invokeImpl(channel, request, 1000).get()).getCause().isInstanceOf(RemotingException.class);
|
|||
|
|
++
|
|||
|
|
++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
|
|||
|
|
++ verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any());
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From b9ffe0f9576f68b8a37cf3e2f68051658ae5a9a2 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Zhouxiang Zhan <zhouxzhan@apache.org>
|
|||
|
|
+Date: Sun, 8 Oct 2023 16:33:44 +0800
|
|||
|
|
+Subject: [PATCH 2/7] [ISSUE #7296] Add ChannelEventListener for
|
|||
|
|
+ MQClientAPIImpl (#7324)
|
|||
|
|
+
|
|||
|
|
+* Add ChannelEventListener for MQClientAPIImpl
|
|||
|
|
+
|
|||
|
|
+* add heartbeat when channel connect
|
|||
|
|
+
|
|||
|
|
+* remove log
|
|||
|
|
+
|
|||
|
|
+* Add enableHeartbeatChannelEventListener for ClientConfig
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/client/ClientConfig.java | 55 ++++++++++++++-----
|
|||
|
|
+ .../rocketmq/client/impl/MQClientAPIImpl.java | 9 ++-
|
|||
|
|
+ .../client/impl/factory/MQClientInstance.java | 35 +++++++++++-
|
|||
|
|
+ .../remoting/netty/NettyRemotingClient.java | 2 +
|
|||
|
|
+ 4 files changed, 85 insertions(+), 16 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
|
|||
|
|
+index bb0fe3522..f9843cc02 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
|
|||
|
|
+@@ -94,6 +94,8 @@ public class ClientConfig {
|
|||
|
|
+ private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false"));
|
|||
|
|
+ private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false"));
|
|||
|
|
+
|
|||
|
|
++ private boolean enableHeartbeatChannelEventListener = true;
|
|||
|
|
++
|
|||
|
|
+ public String buildMQClientId() {
|
|||
|
|
+ StringBuilder sb = new StringBuilder();
|
|||
|
|
+ sb.append(this.getClientIP());
|
|||
|
|
+@@ -201,6 +203,7 @@ public class ClientConfig {
|
|||
|
|
+ this.useHeartbeatV2 = cc.useHeartbeatV2;
|
|||
|
|
+ this.startDetectorEnable = cc.startDetectorEnable;
|
|||
|
|
+ this.sendLatencyEnable = cc.sendLatencyEnable;
|
|||
|
|
++ this.enableHeartbeatChannelEventListener = cc.enableHeartbeatChannelEventListener;
|
|||
|
|
+ this.detectInterval = cc.detectInterval;
|
|||
|
|
+ this.detectTimeout = cc.detectTimeout;
|
|||
|
|
+ }
|
|||
|
|
+@@ -228,6 +231,7 @@ public class ClientConfig {
|
|||
|
|
+ cc.enableStreamRequestType = enableStreamRequestType;
|
|||
|
|
+ cc.useHeartbeatV2 = useHeartbeatV2;
|
|||
|
|
+ cc.startDetectorEnable = startDetectorEnable;
|
|||
|
|
++ cc.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener;
|
|||
|
|
+ cc.sendLatencyEnable = sendLatencyEnable;
|
|||
|
|
+ cc.detectInterval = detectInterval;
|
|||
|
|
+ cc.detectTimeout = detectTimeout;
|
|||
|
|
+@@ -418,6 +422,14 @@ public class ClientConfig {
|
|||
|
|
+ this.startDetectorEnable = startDetectorEnable;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public boolean isEnableHeartbeatChannelEventListener() {
|
|||
|
|
++ return enableHeartbeatChannelEventListener;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setEnableHeartbeatChannelEventListener(boolean enableHeartbeatChannelEventListener) {
|
|||
|
|
++ this.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public int getDetectTimeout() {
|
|||
|
|
+ return this.detectTimeout;
|
|||
|
|
+ }
|
|||
|
|
+@@ -444,19 +456,34 @@ public class ClientConfig {
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String toString() {
|
|||
|
|
+- return "ClientConfig [namesrvAddr=" + namesrvAddr
|
|||
|
|
+- + ", clientIP=" + clientIP + ", instanceName=" + instanceName
|
|||
|
|
+- + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads
|
|||
|
|
+- + ", pollNameServerInterval=" + pollNameServerInterval
|
|||
|
|
+- + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval
|
|||
|
|
+- + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval
|
|||
|
|
+- + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException
|
|||
|
|
+- + ", unitMode=" + unitMode + ", unitName=" + unitName
|
|||
|
|
+- + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS
|
|||
|
|
+- + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name()
|
|||
|
|
+- + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout
|
|||
|
|
+- + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody
|
|||
|
|
+- + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable
|
|||
|
|
+- + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]";
|
|||
|
|
++ return "ClientConfig{" +
|
|||
|
|
++ "namesrvAddr='" + namesrvAddr + '\'' +
|
|||
|
|
++ ", clientIP='" + clientIP + '\'' +
|
|||
|
|
++ ", instanceName='" + instanceName + '\'' +
|
|||
|
|
++ ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads +
|
|||
|
|
++ ", namespace='" + namespace + '\'' +
|
|||
|
|
++ ", namespaceInitialized=" + namespaceInitialized +
|
|||
|
|
++ ", accessChannel=" + accessChannel +
|
|||
|
|
++ ", pollNameServerInterval=" + pollNameServerInterval +
|
|||
|
|
++ ", heartbeatBrokerInterval=" + heartbeatBrokerInterval +
|
|||
|
|
++ ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval +
|
|||
|
|
++ ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException +
|
|||
|
|
++ ", unitMode=" + unitMode +
|
|||
|
|
++ ", unitName='" + unitName + '\'' +
|
|||
|
|
++ ", decodeReadBody=" + decodeReadBody +
|
|||
|
|
++ ", decodeDecompressBody=" + decodeDecompressBody +
|
|||
|
|
++ ", vipChannelEnabled=" + vipChannelEnabled +
|
|||
|
|
++ ", useHeartbeatV2=" + useHeartbeatV2 +
|
|||
|
|
++ ", useTLS=" + useTLS +
|
|||
|
|
++ ", socksProxyConfig='" + socksProxyConfig + '\'' +
|
|||
|
|
++ ", mqClientApiTimeout=" + mqClientApiTimeout +
|
|||
|
|
++ ", detectTimeout=" + detectTimeout +
|
|||
|
|
++ ", detectInterval=" + detectInterval +
|
|||
|
|
++ ", language=" + language +
|
|||
|
|
++ ", enableStreamRequestType=" + enableStreamRequestType +
|
|||
|
|
++ ", sendLatencyEnable=" + sendLatencyEnable +
|
|||
|
|
++ ", startDetectorEnable=" + startDetectorEnable +
|
|||
|
|
++ ", enableHeartbeatChannelEventListener=" + enableHeartbeatChannelEventListener +
|
|||
|
|
++ '}';
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+index 2407e5737..e152be811 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
|
|||
|
|
+@@ -79,6 +79,7 @@ import org.apache.rocketmq.common.sysflag.PullSysFlag;
|
|||
|
|
+ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
++import org.apache.rocketmq.remoting.ChannelEventListener;
|
|||
|
|
+ import org.apache.rocketmq.remoting.CommandCustomHeader;
|
|||
|
|
+ import org.apache.rocketmq.remoting.InvokeCallback;
|
|||
|
|
+ import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+@@ -246,10 +247,16 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
|
|||
|
|
+ public MQClientAPIImpl(final NettyClientConfig nettyClientConfig,
|
|||
|
|
+ final ClientRemotingProcessor clientRemotingProcessor,
|
|||
|
|
+ RPCHook rpcHook, final ClientConfig clientConfig) {
|
|||
|
|
++ this(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public MQClientAPIImpl(final NettyClientConfig nettyClientConfig,
|
|||
|
|
++ final ClientRemotingProcessor clientRemotingProcessor,
|
|||
|
|
++ RPCHook rpcHook, final ClientConfig clientConfig, final ChannelEventListener channelEventListener) {
|
|||
|
|
+ this.clientConfig = clientConfig;
|
|||
|
|
+ topAddressing = new DefaultTopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName());
|
|||
|
|
+ topAddressing.registerChangeCallBack(this);
|
|||
|
|
+- this.remotingClient = new NettyRemotingClient(nettyClientConfig, null);
|
|||
|
|
++ this.remotingClient = new NettyRemotingClient(nettyClientConfig, channelEventListener);
|
|||
|
|
+ this.clientRemotingProcessor = clientRemotingProcessor;
|
|||
|
|
+
|
|||
|
|
+ // Inject stream rpc hook first to make reserve field signature
|
|||
|
|
+diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
|
|||
|
|
+index 9484b26f8..09534a176 100644
|
|||
|
|
+--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
|
|||
|
|
++++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
|
|||
|
|
+@@ -16,6 +16,7 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.client.impl.factory;
|
|||
|
|
+
|
|||
|
|
++import io.netty.channel.Channel;
|
|||
|
|
+ import java.util.Collections;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+@@ -65,6 +66,7 @@ import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueue;
|
|||
|
|
+ import org.apache.rocketmq.common.message.MessageQueueAssignment;
|
|||
|
|
+ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
++import org.apache.rocketmq.remoting.ChannelEventListener;
|
|||
|
|
+ import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+ import org.apache.rocketmq.remoting.common.HeartbeatV2Result;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingException;
|
|||
|
|
+@@ -151,7 +153,38 @@ public class MQClientInstance {
|
|||
|
|
+ this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS());
|
|||
|
|
+ this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig());
|
|||
|
|
+ ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this);
|
|||
|
|
+- this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig);
|
|||
|
|
++ ChannelEventListener channelEventListener;
|
|||
|
|
++ if (clientConfig.isEnableHeartbeatChannelEventListener()) {
|
|||
|
|
++ channelEventListener = new ChannelEventListener() {
|
|||
|
|
++ private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = MQClientInstance.this.brokerAddrTable;
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onChannelConnect(String remoteAddr, Channel channel) {
|
|||
|
|
++ for (Map.Entry<String, HashMap<Long, String>> addressEntry : brokerAddrTable.entrySet()) {
|
|||
|
|
++ for (String address : addressEntry.getValue().values()) {
|
|||
|
|
++ if (address.equals(remoteAddr)) {
|
|||
|
|
++ sendHeartbeatToAllBrokerWithLockV2(false);
|
|||
|
|
++ break;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onChannelClose(String remoteAddr, Channel channel) {
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onChannelException(String remoteAddr, Channel channel) {
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public void onChannelIdle(String remoteAddr, Channel channel) {
|
|||
|
|
++ }
|
|||
|
|
++ };
|
|||
|
|
++ } else {
|
|||
|
|
++ channelEventListener = null;
|
|||
|
|
++ }
|
|||
|
|
++ this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, channelEventListener);
|
|||
|
|
+
|
|||
|
|
+ if (this.clientConfig.getNamesrvAddr() != null) {
|
|||
|
|
+ this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr());
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
+index d784351a5..8631d0447 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
+@@ -229,6 +229,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ handler.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ nettyEventExecutor.start();
|
|||
|
|
++
|
|||
|
|
+ TimerTask timerTaskScanResponseTable = new TimerTask() {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void run(Timeout timeout) {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 3808387e1389278edbe4ef023d200ecb3015622b Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: lk <xdkxlk@outlook.com>
|
|||
|
|
+Date: Mon, 9 Oct 2023 16:07:56 +0800
|
|||
|
|
+Subject: [PATCH 3/7] [ISSUE #7429] clean channel map when CLIENT_UNREGISTER in
|
|||
|
|
+ proxy
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../service/sysmessage/HeartbeatSyncer.java | 31 ++++++---
|
|||
|
|
+ .../sysmessage/HeartbeatSyncerTest.java | 68 +++++++++++++++++++
|
|||
|
|
+ 2 files changed, 88 insertions(+), 11 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
|
|||
|
|
+index f70c06b8f..fee3ea87d 100644
|
|||
|
|
+--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
|
|||
|
|
++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
|
|||
|
|
+@@ -18,6 +18,7 @@
|
|||
|
|
+ package org.apache.rocketmq.proxy.service.sysmessage;
|
|||
|
|
+
|
|||
|
|
+ import com.alibaba.fastjson.JSON;
|
|||
|
|
++import io.netty.channel.Channel;
|
|||
|
|
+ import java.nio.charset.StandardCharsets;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+ import java.util.Map;
|
|||
|
|
+@@ -73,16 +74,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
|
|||
|
|
+ );
|
|||
|
|
+ this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() {
|
|||
|
|
+ @Override
|
|||
|
|
+- public void handle(ConsumerGroupEvent event, String s, Object... args) {
|
|||
|
|
+- if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {
|
|||
|
|
+- if (args == null || args.length < 1) {
|
|||
|
|
+- return;
|
|||
|
|
+- }
|
|||
|
|
+- if (args[0] instanceof ClientChannelInfo) {
|
|||
|
|
+- ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
|
|||
|
|
+- remoteChannelMap.remove(clientChannelInfo.getChannel().id().asLongText());
|
|||
|
|
+- }
|
|||
|
|
+- }
|
|||
|
|
++ public void handle(ConsumerGroupEvent event, String group, Object... args) {
|
|||
|
|
++ processConsumerGroupEvent(event, group, args);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+@@ -98,6 +91,18 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
|
|||
|
|
+ super.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ protected void processConsumerGroupEvent(ConsumerGroupEvent event, String group, Object... args) {
|
|||
|
|
++ if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {
|
|||
|
|
++ if (args == null || args.length < 1) {
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ if (args[0] instanceof ClientChannelInfo) {
|
|||
|
|
++ ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
|
|||
|
|
++ remoteChannelMap.remove(buildKey(group, clientChannelInfo.getChannel()));
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo,
|
|||
|
|
+ ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
|
|||
|
|
+ Set<SubscriptionData> subList) {
|
|||
|
|
+@@ -189,7 +194,7 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData());
|
|||
|
|
+- RemoteChannel channel = remoteChannelMap.computeIfAbsent(data.getGroup() + "@" + decodedChannel.id().asLongText(), key -> decodedChannel);
|
|||
|
|
++ RemoteChannel channel = remoteChannelMap.computeIfAbsent(buildKey(data.getGroup(), decodedChannel), key -> decodedChannel);
|
|||
|
|
+ channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute());
|
|||
|
|
+ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
|
|||
|
|
+ channel,
|
|||
|
|
+@@ -228,4 +233,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
|
|||
|
|
+ // use local address, remoting port and grpc port to build unique local proxy Id
|
|||
|
|
+ return proxyConfig.getLocalServeAddr() + "%" + proxyConfig.getRemotingListenPort() + "%" + proxyConfig.getGrpcServerPort();
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ private static String buildKey(String group, Channel channel) {
|
|||
|
|
++ return group + "@" + channel.id().asLongText();
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
|
|||
|
|
+index 43fba3d03..9a2c5e343 100644
|
|||
|
|
+--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
|
|||
|
|
++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
|
|||
|
|
+@@ -27,6 +27,7 @@ import com.google.common.collect.Sets;
|
|||
|
|
+ import io.netty.channel.Channel;
|
|||
|
|
+ import io.netty.channel.ChannelId;
|
|||
|
|
+ import java.time.Duration;
|
|||
|
|
++import java.util.Collections;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+ import java.util.HashSet;
|
|||
|
|
+ import java.util.List;
|
|||
|
|
+@@ -35,6 +36,7 @@ import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.stream.Collectors;
|
|||
|
|
+ import org.apache.commons.lang3.RandomStringUtils;
|
|||
|
|
+ import org.apache.rocketmq.broker.client.ClientChannelInfo;
|
|||
|
|
++import org.apache.rocketmq.broker.client.ConsumerGroupEvent;
|
|||
|
|
+ import org.apache.rocketmq.broker.client.ConsumerManager;
|
|||
|
|
+ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;
|
|||
|
|
+ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
|
|||
|
|
+@@ -320,6 +322,72 @@ public class HeartbeatSyncerTest extends InitConfigTest {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testProcessConsumerGroupEventForRemoting() {
|
|||
|
|
++ String consumerGroup = "consumerGroup";
|
|||
|
|
++ Channel channel = createMockChannel();
|
|||
|
|
++ RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class);
|
|||
|
|
++ RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, Collections.emptySet());
|
|||
|
|
++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
|
|||
|
|
++ remotingChannel,
|
|||
|
|
++ clientId,
|
|||
|
|
++ LanguageCode.JAVA,
|
|||
|
|
++ 4
|
|||
|
|
++ );
|
|||
|
|
++
|
|||
|
|
++ testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Test
|
|||
|
|
++ public void testProcessConsumerGroupEventForGrpcV2() {
|
|||
|
|
++ String consumerGroup = "consumerGroup";
|
|||
|
|
++ GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class);
|
|||
|
|
++ GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class);
|
|||
|
|
++ GrpcClientChannel grpcClientChannel = new GrpcClientChannel(
|
|||
|
|
++ proxyRelayService, grpcClientSettingsManager, grpcChannelManager,
|
|||
|
|
++ ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress),
|
|||
|
|
++ clientId);
|
|||
|
|
++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
|
|||
|
|
++ grpcClientChannel,
|
|||
|
|
++ clientId,
|
|||
|
|
++ LanguageCode.JAVA,
|
|||
|
|
++ 5
|
|||
|
|
++ );
|
|||
|
|
++
|
|||
|
|
++ testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo);
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ private void testProcessConsumerGroupEvent(String consumerGroup, ClientChannelInfo clientChannelInfo) {
|
|||
|
|
++ HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null);
|
|||
|
|
++ SendResult okSendResult = new SendResult();
|
|||
|
|
++ okSendResult.setSendStatus(SendStatus.SEND_OK);
|
|||
|
|
++
|
|||
|
|
++ ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);
|
|||
|
|
++ doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt)
|
|||
|
|
++ .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong());
|
|||
|
|
++
|
|||
|
|
++ heartbeatSyncer.onConsumerRegister(
|
|||
|
|
++ consumerGroup,
|
|||
|
|
++ clientChannelInfo,
|
|||
|
|
++ ConsumeType.CONSUME_PASSIVELY,
|
|||
|
|
++ MessageModel.CLUSTERING,
|
|||
|
|
++ ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,
|
|||
|
|
++ Collections.emptySet()
|
|||
|
|
++ );
|
|||
|
|
++ await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 1);
|
|||
|
|
++
|
|||
|
|
++ // change local serve addr, to simulate other proxy receive messages
|
|||
|
|
++ heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);
|
|||
|
|
++ ArgumentCaptor<ClientChannelInfo> channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);
|
|||
|
|
++ doReturn(true).when(consumerManager).registerConsumer(anyString(), channelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());
|
|||
|
|
++
|
|||
|
|
++ heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null);
|
|||
|
|
++ assertEquals(1, heartbeatSyncer.remoteChannelMap.size());
|
|||
|
|
++
|
|||
|
|
++ heartbeatSyncer.processConsumerGroupEvent(ConsumerGroupEvent.CLIENT_UNREGISTER, consumerGroup, channelInfoArgumentCaptor.getValue());
|
|||
|
|
++ assertTrue(heartbeatSyncer.remoteChannelMap.isEmpty());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ private MessageExt convertFromMessage(Message message) {
|
|||
|
|
+ MessageExt messageExt = new MessageExt();
|
|||
|
|
+ messageExt.setTopic(message.getTopic());
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 0027a1486d4f2d6f7dce3010751167e883783945 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: redlsz <szliu0927@gmail.com>
|
|||
|
|
+Date: Mon, 9 Oct 2023 16:52:10 +0800
|
|||
|
|
+Subject: [PATCH 4/7] [ISSUE #7412] Fix pop revive message error when reput
|
|||
|
|
+ checkpoint
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../org/apache/rocketmq/broker/processor/PopReviveService.java | 1 +
|
|||
|
|
+ 1 file changed, 1 insertion(+)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
|
|||
|
|
+index 93167db37..d5174d3d1 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
|
|||
|
|
+@@ -595,6 +595,7 @@ public class PopReviveService extends ServiceThread {
|
|||
|
|
+ newCk.setCId(oldCK.getCId());
|
|||
|
|
+ newCk.setTopic(oldCK.getTopic());
|
|||
|
|
+ newCk.setQueueId(oldCK.getQueueId());
|
|||
|
|
++ newCk.setBrokerName(oldCK.getBrokerName());
|
|||
|
|
+ newCk.addDiff(0);
|
|||
|
|
+ MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId);
|
|||
|
|
+ brokerController.getMessageStore().putMessage(ckMsg);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From b18e564addbcff50165a5e1d9d4ab7db789d901b Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: rongtong <jinrongtong5@163.com>
|
|||
|
|
+Date: Mon, 9 Oct 2023 21:43:01 +0800
|
|||
|
|
+Subject: [PATCH 5/7] [ISSUE #7431] Fix flaky test of
|
|||
|
|
+ DLedgerControllerTest#testBrokerLifecycleListener (#7432)
|
|||
|
|
+
|
|||
|
|
+* Fix flaky test of DLedgerControllerTest#testBrokerLifecycleListener
|
|||
|
|
+---
|
|||
|
|
+ .../impl/DLedgerControllerTest.java | 26 ++++++++++++-------
|
|||
|
|
+ 1 file changed, 17 insertions(+), 9 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java
|
|||
|
|
+index 595a5cb65..d6e5449c5 100644
|
|||
|
|
+--- a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java
|
|||
|
|
++++ b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java
|
|||
|
|
+@@ -63,7 +63,8 @@ public class DLedgerControllerTest {
|
|||
|
|
+ private List<String> baseDirs;
|
|||
|
|
+ private List<DLedgerController> controllers;
|
|||
|
|
+
|
|||
|
|
+- public DLedgerController launchController(final String group, final String peers, final String selfId, final boolean isEnableElectUncleanMaster) {
|
|||
|
|
++ public DLedgerController launchController(final String group, final String peers, final String selfId,
|
|||
|
|
++ final boolean isEnableElectUncleanMaster) {
|
|||
|
|
+ String tmpdir = System.getProperty("java.io.tmpdir");
|
|||
|
|
+ final String path = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + group + File.separator + selfId;
|
|||
|
|
+ baseDirs.add(path);
|
|||
|
|
+@@ -121,11 +122,11 @@ public class DLedgerControllerTest {
|
|||
|
|
+ final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, nextBrokerId, brokerAddress);
|
|||
|
|
+ RemotingCommand remotingCommand2 = leader.registerBroker(registerBrokerToControllerRequestHeader).get(2, TimeUnit.SECONDS);
|
|||
|
|
+
|
|||
|
|
+-
|
|||
|
|
+ assertEquals(ResponseCode.SUCCESS, remotingCommand2.getCode());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, Long brokerId,
|
|||
|
|
++ public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress,
|
|||
|
|
++ Long brokerId,
|
|||
|
|
+ boolean exceptSuccess) throws Exception {
|
|||
|
|
+ final ElectMasterRequestHeader electMasterRequestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId);
|
|||
|
|
+ RemotingCommand command = leader.electMaster(electMasterRequestHeader).get(2, TimeUnit.SECONDS);
|
|||
|
|
+@@ -186,9 +187,9 @@ public class DLedgerControllerTest {
|
|||
|
|
+ registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L);
|
|||
|
|
+ registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L);
|
|||
|
|
+ // try elect
|
|||
|
|
+- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L,true);
|
|||
|
|
+- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false);
|
|||
|
|
+- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L,false);
|
|||
|
|
++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, true);
|
|||
|
|
++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false);
|
|||
|
|
++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, false);
|
|||
|
|
+ final RemotingCommand getInfoResponse = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS);
|
|||
|
|
+ final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) getInfoResponse.readCustomHeader();
|
|||
|
|
+ assertEquals(1, replicaInfo.getMasterEpoch().intValue());
|
|||
|
|
+@@ -239,6 +240,8 @@ public class DLedgerControllerTest {
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testBrokerLifecycleListener() throws Exception {
|
|||
|
|
+ final DLedgerController leader = mockMetaData(false);
|
|||
|
|
++
|
|||
|
|
++ assertTrue(leader.isLeaderState());
|
|||
|
|
+ // Mock that master broker has been inactive, and try to elect a new master from sync-state-set
|
|||
|
|
+ // But we shut down two controller, so the ElectMasterEvent will be appended to DLedger failed.
|
|||
|
|
+ // So the statemachine still keep the stale master's information
|
|||
|
|
+@@ -247,15 +250,20 @@ public class DLedgerControllerTest {
|
|||
|
|
+ dLedgerController.shutdown();
|
|||
|
|
+ controllers.remove(dLedgerController);
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
+ final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);
|
|||
|
|
+ setBrokerElectPolicy(leader, 1L);
|
|||
|
|
+ Exception exception = null;
|
|||
|
|
++ RemotingCommand remotingCommand = null;
|
|||
|
|
+ try {
|
|||
|
|
+- leader.electMaster(request).get(5, TimeUnit.SECONDS);
|
|||
|
|
++ remotingCommand = leader.electMaster(request).get(5, TimeUnit.SECONDS);
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ exception = e;
|
|||
|
|
+ }
|
|||
|
|
+- assertNotNull(exception);
|
|||
|
|
++
|
|||
|
|
++ assertTrue(exception != null ||
|
|||
|
|
++ remotingCommand != null && remotingCommand.getCode() == ResponseCode.CONTROLLER_NOT_LEADER);
|
|||
|
|
++
|
|||
|
|
+ // Shut down leader controller
|
|||
|
|
+ leader.shutdown();
|
|||
|
|
+ controllers.remove(leader);
|
|||
|
|
+@@ -272,7 +280,7 @@ public class DLedgerControllerTest {
|
|||
|
|
+ setBrokerAlivePredicate(newLeader, 1L);
|
|||
|
|
+ // Check if the statemachine is stale
|
|||
|
|
+ final RemotingCommand resp = newLeader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).
|
|||
|
|
+- get(10, TimeUnit.SECONDS);
|
|||
|
|
++ get(10, TimeUnit.SECONDS);
|
|||
|
|
+ final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader();
|
|||
|
|
+ assertEquals(1, replicaInfo.getMasterBrokerId().longValue());
|
|||
|
|
+ assertEquals(1, replicaInfo.getMasterEpoch().intValue());
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 38d3d5d95d371ac89f7d491a4c8719b4a22c60e1 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: mxsm <ljbmxsm@gmail.com>
|
|||
|
|
+Date: Tue, 10 Oct 2023 09:37:04 +0800
|
|||
|
|
+Subject: [PATCH 6/7] [ISSUE #7433]Update the version in the README.md document
|
|||
|
|
+ to 5.1.4 (#7434)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ README.md | 8 ++++----
|
|||
|
|
+ 1 file changed, 4 insertions(+), 4 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/README.md b/README.md
|
|||
|
|
+index 56d253ce1..5aaa2ba73 100644
|
|||
|
|
+--- a/README.md
|
|||
|
|
++++ b/README.md
|
|||
|
|
+@@ -49,21 +49,21 @@ $ java -version
|
|||
|
|
+ java version "1.8.0_121"
|
|||
|
|
+ ```
|
|||
|
|
+
|
|||
|
|
+-For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip) to download the 5.1.3 RocketMQ binary release,
|
|||
|
|
++For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip) to download the 5.1.4 RocketMQ binary release,
|
|||
|
|
+ unpack it to your local disk, such as `D:\rocketmq`.
|
|||
|
|
+ For macOS and Linux users, execute following commands:
|
|||
|
|
+
|
|||
|
|
+ ```shell
|
|||
|
|
+ # Download release from the Apache mirror
|
|||
|
|
+-$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip
|
|||
|
|
++$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip
|
|||
|
|
+
|
|||
|
|
+ # Unpack the release
|
|||
|
|
+-$ unzip rocketmq-all-5.1.3-bin-release.zip
|
|||
|
|
++$ unzip rocketmq-all-5.1.4-bin-release.zip
|
|||
|
|
+ ```
|
|||
|
|
+
|
|||
|
|
+ Prepare a terminal and change to the extracted `bin` directory:
|
|||
|
|
+ ```shell
|
|||
|
|
+-$ cd rocketmq-all-5.1.3-bin-release/bin
|
|||
|
|
++$ cd rocketmq-all-5.1.4-bin-release/bin
|
|||
|
|
+ ```
|
|||
|
|
+
|
|||
|
|
+ **1) Start NameServer**
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 4acb43ecee03e429d036e3ff4c28bd402d1b30c7 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Zhouxiang Zhan <zhouxzhan@apache.org>
|
|||
|
|
+Date: Tue, 10 Oct 2023 13:54:01 +0800
|
|||
|
|
+Subject: [PATCH 7/7] [ISSUE #7330] Add goaway and reconnection mechanism
|
|||
|
|
+ (#7331)
|
|||
|
|
+
|
|||
|
|
+* Add shutdown wait for NettyRemotingServer
|
|||
|
|
+
|
|||
|
|
+* Add goaway and reconnection mechanism
|
|||
|
|
+
|
|||
|
|
+* Add client version check
|
|||
|
|
+
|
|||
|
|
+* Add enableTransparentRetry for NettyClientConfig
|
|||
|
|
+
|
|||
|
|
+* Add enableReconnectForGoAway for NettyClientConfig
|
|||
|
|
+
|
|||
|
|
+* fix unit test
|
|||
|
|
+
|
|||
|
|
+* fix client version check
|
|||
|
|
+---
|
|||
|
|
+ .../remoting/netty/NettyClientConfig.java | 30 ++++
|
|||
|
|
+ .../remoting/netty/NettyRemotingAbstract.java | 15 ++
|
|||
|
|
+ .../remoting/netty/NettyRemotingClient.java | 153 ++++++++++++++++--
|
|||
|
|
+ .../remoting/netty/NettyRemotingServer.java | 31 ++--
|
|||
|
|
+ .../remoting/netty/NettyServerConfig.java | 19 +++
|
|||
|
|
+ .../remoting/protocol/ResponseCode.java | 2 +
|
|||
|
|
+ .../netty/NettyRemotingClientTest.java | 39 ++---
|
|||
|
|
+ 7 files changed, 239 insertions(+), 50 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
|
|||
|
|
+index b2e7df754..c28288786 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
|
|||
|
|
+@@ -53,6 +53,12 @@ public class NettyClientConfig {
|
|||
|
|
+ private boolean disableCallbackExecutor = false;
|
|||
|
|
+ private boolean disableNettyWorkerGroup = false;
|
|||
|
|
+
|
|||
|
|
++ private long maxReconnectIntervalTimeSeconds = 60;
|
|||
|
|
++
|
|||
|
|
++ private boolean enableReconnectForGoAway = true;
|
|||
|
|
++
|
|||
|
|
++ private boolean enableTransparentRetry = true;
|
|||
|
|
++
|
|||
|
|
+ public boolean isClientCloseSocketIfTimeout() {
|
|||
|
|
+ return clientCloseSocketIfTimeout;
|
|||
|
|
+ }
|
|||
|
|
+@@ -181,6 +187,30 @@ public class NettyClientConfig {
|
|||
|
|
+ this.disableNettyWorkerGroup = disableNettyWorkerGroup;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ public long getMaxReconnectIntervalTimeSeconds() {
|
|||
|
|
++ return maxReconnectIntervalTimeSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setMaxReconnectIntervalTimeSeconds(long maxReconnectIntervalTimeSeconds) {
|
|||
|
|
++ this.maxReconnectIntervalTimeSeconds = maxReconnectIntervalTimeSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isEnableReconnectForGoAway() {
|
|||
|
|
++ return enableReconnectForGoAway;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) {
|
|||
|
|
++ this.enableReconnectForGoAway = enableReconnectForGoAway;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isEnableTransparentRetry() {
|
|||
|
|
++ return enableTransparentRetry;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setEnableTransparentRetry(boolean enableTransparentRetry) {
|
|||
|
|
++ this.enableTransparentRetry = enableTransparentRetry;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public String getSocksProxyConfig() {
|
|||
|
|
+ return socksProxyConfig;
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
|
|||
|
|
+index 12e66f913..07ace28ea 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
|
|||
|
|
+@@ -40,9 +40,11 @@ import java.util.concurrent.Semaphore;
|
|||
|
|
+ import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import java.util.concurrent.TimeoutException;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicReference;
|
|||
|
|
++import java.util.concurrent.atomic.AtomicBoolean;
|
|||
|
|
+ import java.util.function.Consumer;
|
|||
|
|
+ import javax.annotation.Nullable;
|
|||
|
|
+ import org.apache.rocketmq.common.AbortProcessException;
|
|||
|
|
++import org.apache.rocketmq.common.MQVersion;
|
|||
|
|
+ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.ServiceThread;
|
|||
|
|
+ import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
+@@ -60,6 +62,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.ResponseCode;
|
|||
|
|
+
|
|||
|
|
+ import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_IS_LONG_POLLING;
|
|||
|
|
+ import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE;
|
|||
|
|
+@@ -120,6 +123,8 @@ public abstract class NettyRemotingAbstract {
|
|||
|
|
+ */
|
|||
|
|
+ protected List<RPCHook> rpcHooks = new ArrayList<>();
|
|||
|
|
+
|
|||
|
|
++ protected AtomicBoolean isShuttingDown = new AtomicBoolean(false);
|
|||
|
|
++
|
|||
|
|
+ static {
|
|||
|
|
+ NettyLogger.initNettyLogger();
|
|||
|
|
+ }
|
|||
|
|
+@@ -264,6 +269,16 @@ public abstract class NettyRemotingAbstract {
|
|||
|
|
+
|
|||
|
|
+ Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque);
|
|||
|
|
+
|
|||
|
|
++ if (isShuttingDown.get()) {
|
|||
|
|
++ if (cmd.getVersion() > MQVersion.Version.V5_1_4.ordinal()) {
|
|||
|
|
++ final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY,
|
|||
|
|
++ "please go away");
|
|||
|
|
++ response.setOpaque(opaque);
|
|||
|
|
++ writeResponse(ctx.channel(), cmd, response);
|
|||
|
|
++ return;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ if (pair.getObject1().rejectRequest()) {
|
|||
|
|
+ final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY,
|
|||
|
|
+ "[REJECTREQUEST]system busy, start flow control for a while");
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
+index 8631d0447..4bc51bd83 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
|
|||
|
|
+@@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty;
|
|||
|
|
+
|
|||
|
|
+ import com.alibaba.fastjson.JSON;
|
|||
|
|
+ import com.alibaba.fastjson.TypeReference;
|
|||
|
|
++import com.google.common.base.Stopwatch;
|
|||
|
|
+ import io.netty.bootstrap.Bootstrap;
|
|||
|
|
+ import io.netty.buffer.PooledByteBufAllocator;
|
|||
|
|
+ import io.netty.channel.Channel;
|
|||
|
|
+@@ -48,6 +49,7 @@ import java.io.IOException;
|
|||
|
|
+ import java.net.InetSocketAddress;
|
|||
|
|
+ import java.net.SocketAddress;
|
|||
|
|
+ import java.security.cert.CertificateException;
|
|||
|
|
++import java.time.Duration;
|
|||
|
|
+ import java.util.ArrayList;
|
|||
|
|
+ import java.util.Collections;
|
|||
|
|
+ import java.util.HashMap;
|
|||
|
|
+@@ -57,6 +59,7 @@ import java.util.Map;
|
|||
|
|
+ import java.util.Random;
|
|||
|
|
+ import java.util.Set;
|
|||
|
|
+ import java.util.concurrent.ArrayBlockingQueue;
|
|||
|
|
++import java.util.concurrent.CompletableFuture;
|
|||
|
|
+ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+ import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+ import java.util.concurrent.ExecutorService;
|
|||
|
|
+@@ -66,6 +69,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
+ import java.util.concurrent.atomic.AtomicReference;
|
|||
|
|
+ import java.util.concurrent.locks.Lock;
|
|||
|
|
+ import java.util.concurrent.locks.ReentrantLock;
|
|||
|
|
++import java.util.concurrent.locks.ReentrantReadWriteLock;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+ import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+@@ -82,6 +86,7 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
++import org.apache.rocketmq.remoting.protocol.ResponseCode;
|
|||
|
|
+ import org.apache.rocketmq.remoting.proxy.SocksProxyConfig;
|
|||
|
|
+
|
|||
|
|
+ public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient {
|
|||
|
|
+@@ -97,6 +102,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ private final Map<String /* cidr */, SocksProxyConfig /* proxy */> proxyMap = new HashMap<>();
|
|||
|
|
+ private final ConcurrentHashMap<String /* cidr */, Bootstrap> bootstrapMap = new ConcurrentHashMap<>();
|
|||
|
|
+ private final ConcurrentMap<String /* addr */, ChannelWrapper> channelTables = new ConcurrentHashMap<>();
|
|||
|
|
++ private final ConcurrentMap<Channel, ChannelWrapper> channelWrapperTables = new ConcurrentHashMap<>();
|
|||
|
|
+
|
|||
|
|
+ private final HashedWheelTimer timer = new HashedWheelTimer(r -> new Thread(r, "ClientHouseKeepingService"));
|
|||
|
|
+
|
|||
|
|
+@@ -356,9 +362,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ this.timer.stop();
|
|||
|
|
+
|
|||
|
|
+ for (String addr : this.channelTables.keySet()) {
|
|||
|
|
+- this.closeChannel(addr, this.channelTables.get(addr).getChannel());
|
|||
|
|
++ this.channelTables.get(addr).close();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ this.channelWrapperTables.clear();
|
|||
|
|
+ this.channelTables.clear();
|
|||
|
|
+
|
|||
|
|
+ this.eventLoopGroupWorker.shutdownGracefully();
|
|||
|
|
+@@ -416,7 +423,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (removeItemFromTable) {
|
|||
|
|
+- this.channelTables.remove(addrRemote);
|
|||
|
|
++ ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel);
|
|||
|
|
++ if (channelWrapper != null && channelWrapper.tryClose(channel)) {
|
|||
|
|
++ this.channelTables.remove(addrRemote);
|
|||
|
|
++ }
|
|||
|
|
+ LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+@@ -463,7 +473,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (removeItemFromTable) {
|
|||
|
|
+- this.channelTables.remove(addrRemote);
|
|||
|
|
++ ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel);
|
|||
|
|
++ if (channelWrapper != null && channelWrapper.tryClose(channel)) {
|
|||
|
|
++ this.channelTables.remove(addrRemote);
|
|||
|
|
++ }
|
|||
|
|
+ LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote);
|
|||
|
|
+ RemotingHelper.closeChannel(channel);
|
|||
|
|
+ }
|
|||
|
|
+@@ -511,7 +524,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ if (addr.contains(namesrvAddr)) {
|
|||
|
|
+ ChannelWrapper channelWrapper = this.channelTables.get(addr);
|
|||
|
|
+ if (channelWrapper != null) {
|
|||
|
|
+- closeChannel(channelWrapper.getChannel());
|
|||
|
|
++ channelWrapper.close();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+@@ -689,8 +702,9 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ ChannelFuture channelFuture = fetchBootstrap(addr)
|
|||
|
|
+ .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1]));
|
|||
|
|
+ LOGGER.info("createChannel: begin to connect remote host[{}] asynchronously", addr);
|
|||
|
|
+- cw = new ChannelWrapper(channelFuture);
|
|||
|
|
++ cw = new ChannelWrapper(addr, channelFuture);
|
|||
|
|
+ this.channelTables.put(addr, cw);
|
|||
|
|
++ this.channelWrapperTables.put(channelFuture.channel(), cw);
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ LOGGER.error("createChannel: create channel exception", e);
|
|||
|
|
+@@ -758,6 +772,64 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
++ @Override
|
|||
|
|
++ public CompletableFuture<RemotingCommand> invoke(String addr, RemotingCommand request,
|
|||
|
|
++ long timeoutMillis) {
|
|||
|
|
++ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
|
|||
|
|
++ try {
|
|||
|
|
++ final Channel channel = this.getAndCreateChannel(addr);
|
|||
|
|
++ if (channel != null && channel.isActive()) {
|
|||
|
|
++ return invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> {
|
|||
|
|
++ if (t == null) {
|
|||
|
|
++ updateChannelLastResponseTime(addr);
|
|||
|
|
++ }
|
|||
|
|
++ }).thenApply(ResponseFuture::getResponseCommand);
|
|||
|
|
++ } else {
|
|||
|
|
++ this.closeChannel(addr, channel);
|
|||
|
|
++ future.completeExceptionally(new RemotingConnectException(addr));
|
|||
|
|
++ }
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ future.completeExceptionally(t);
|
|||
|
|
++ }
|
|||
|
|
++ return future;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request,
|
|||
|
|
++ final long timeoutMillis) {
|
|||
|
|
++ Stopwatch stopwatch = Stopwatch.createStarted();
|
|||
|
|
++ return super.invokeImpl(channel, request, timeoutMillis).thenCompose(responseFuture -> {
|
|||
|
|
++ RemotingCommand response = responseFuture.getResponseCommand();
|
|||
|
|
++ if (response.getCode() == ResponseCode.GO_AWAY) {
|
|||
|
|
++ if (nettyClientConfig.isEnableReconnectForGoAway()) {
|
|||
|
|
++ ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> {
|
|||
|
|
++ try {
|
|||
|
|
++ if (channelWrapper0.reconnect()) {
|
|||
|
|
++ LOGGER.info("Receive go away from channel {}, recreate the channel", channel0);
|
|||
|
|
++ channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0);
|
|||
|
|
++ }
|
|||
|
|
++ } catch (Throwable t) {
|
|||
|
|
++ LOGGER.error("Channel {} reconnect error", channelWrapper0, t);
|
|||
|
|
++ }
|
|||
|
|
++ return channelWrapper0;
|
|||
|
|
++ });
|
|||
|
|
++ if (channelWrapper != null) {
|
|||
|
|
++ if (nettyClientConfig.isEnableTransparentRetry()) {
|
|||
|
|
++ long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS);
|
|||
|
|
++ stopwatch.stop();
|
|||
|
|
++ RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader());
|
|||
|
|
++ Channel retryChannel = channelWrapper.getChannel();
|
|||
|
|
++ if (channel != retryChannel) {
|
|||
|
|
++ return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ return CompletableFuture.completedFuture(responseFuture);
|
|||
|
|
++ });
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ @Override
|
|||
|
|
+ public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) {
|
|||
|
|
+ ExecutorService executorThis = executor;
|
|||
|
|
+@@ -877,30 +949,41 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- static class ChannelWrapper {
|
|||
|
|
+- private final ChannelFuture channelFuture;
|
|||
|
|
++ class ChannelWrapper {
|
|||
|
|
++ private final ReentrantReadWriteLock lock;
|
|||
|
|
++ private ChannelFuture channelFuture;
|
|||
|
|
+ // only affected by sync or async request, oneway is not included.
|
|||
|
|
++ private ChannelFuture channelToClose;
|
|||
|
|
+ private long lastResponseTime;
|
|||
|
|
++ private volatile long lastReconnectTimestamp = 0L;
|
|||
|
|
++ private final String channelAddress;
|
|||
|
|
+
|
|||
|
|
+- public ChannelWrapper(ChannelFuture channelFuture) {
|
|||
|
|
++ public ChannelWrapper(String address, ChannelFuture channelFuture) {
|
|||
|
|
++ this.lock = new ReentrantReadWriteLock();
|
|||
|
|
+ this.channelFuture = channelFuture;
|
|||
|
|
+ this.lastResponseTime = System.currentTimeMillis();
|
|||
|
|
++ this.channelAddress = address;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public boolean isOK() {
|
|||
|
|
+- return this.channelFuture.channel() != null && this.channelFuture.channel().isActive();
|
|||
|
|
++ return getChannel() != null && getChannel().isActive();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public boolean isWritable() {
|
|||
|
|
+- return this.channelFuture.channel().isWritable();
|
|||
|
|
++ return getChannel().isWritable();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private Channel getChannel() {
|
|||
|
|
+- return this.channelFuture.channel();
|
|||
|
|
++ return getChannelFuture().channel();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public ChannelFuture getChannelFuture() {
|
|||
|
|
+- return channelFuture;
|
|||
|
|
++ lock.readLock().lock();
|
|||
|
|
++ try {
|
|||
|
|
++ return this.channelFuture;
|
|||
|
|
++ } finally {
|
|||
|
|
++ lock.readLock().unlock();
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long getLastResponseTime() {
|
|||
|
|
+@@ -910,6 +993,52 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ public void updateLastResponseTime() {
|
|||
|
|
+ this.lastResponseTime = System.currentTimeMillis();
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public boolean reconnect() {
|
|||
|
|
++ if (lock.writeLock().tryLock()) {
|
|||
|
|
++ try {
|
|||
|
|
++ if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) {
|
|||
|
|
++ channelToClose = channelFuture;
|
|||
|
|
++ String[] hostAndPort = getHostAndPort(channelAddress);
|
|||
|
|
++ channelFuture = fetchBootstrap(channelAddress)
|
|||
|
|
++ .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1]));
|
|||
|
|
++ lastReconnectTimestamp = System.currentTimeMillis();
|
|||
|
|
++ return true;
|
|||
|
|
++ }
|
|||
|
|
++ } finally {
|
|||
|
|
++ lock.writeLock().unlock();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public boolean tryClose(Channel channel) {
|
|||
|
|
++ try {
|
|||
|
|
++ lock.readLock().lock();
|
|||
|
|
++ if (channelFuture != null) {
|
|||
|
|
++ if (channelFuture.channel().equals(channel)) {
|
|||
|
|
++ return true;
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ } finally {
|
|||
|
|
++ lock.readLock().unlock();
|
|||
|
|
++ }
|
|||
|
|
++ return false;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void close() {
|
|||
|
|
++ try {
|
|||
|
|
++ lock.writeLock().lock();
|
|||
|
|
++ if (channelFuture != null) {
|
|||
|
|
++ closeChannel(channelFuture.channel());
|
|||
|
|
++ }
|
|||
|
|
++ if (channelToClose != null) {
|
|||
|
|
++ closeChannel(channelToClose.channel());
|
|||
|
|
++ }
|
|||
|
|
++ } finally {
|
|||
|
|
++ lock.writeLock().unlock();
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ class InvokeCallbackWrapper implements InvokeCallback {
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
+index aa0d46542..735d36168 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
|
|||
|
|
+@@ -53,6 +53,19 @@ import io.netty.util.HashedWheelTimer;
|
|||
|
|
+ import io.netty.util.Timeout;
|
|||
|
|
+ import io.netty.util.TimerTask;
|
|||
|
|
+ import io.netty.util.concurrent.DefaultEventExecutorGroup;
|
|||
|
|
++import java.io.IOException;
|
|||
|
|
++import java.net.InetSocketAddress;
|
|||
|
|
++import java.security.cert.CertificateException;
|
|||
|
|
++import java.time.Duration;
|
|||
|
|
++import java.util.List;
|
|||
|
|
++import java.util.NoSuchElementException;
|
|||
|
|
++import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
++import java.util.concurrent.ConcurrentMap;
|
|||
|
|
++import java.util.concurrent.ExecutorService;
|
|||
|
|
++import java.util.concurrent.Executors;
|
|||
|
|
++import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
++import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
++import java.util.concurrent.TimeUnit;
|
|||
|
|
+ import org.apache.commons.collections.CollectionUtils;
|
|||
|
|
+ import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.rocketmq.common.Pair;
|
|||
|
|
+@@ -74,19 +87,6 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
|
|||
|
|
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
|
|||
|
|
+
|
|||
|
|
+-import java.io.IOException;
|
|||
|
|
+-import java.net.InetSocketAddress;
|
|||
|
|
+-import java.security.cert.CertificateException;
|
|||
|
|
+-import java.util.List;
|
|||
|
|
+-import java.util.NoSuchElementException;
|
|||
|
|
+-import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+-import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+-import java.util.concurrent.ExecutorService;
|
|||
|
|
+-import java.util.concurrent.Executors;
|
|||
|
|
+-import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+-import java.util.concurrent.ThreadPoolExecutor;
|
|||
|
|
+-import java.util.concurrent.TimeUnit;
|
|||
|
|
+-
|
|||
|
|
+ @SuppressWarnings("NullableProblems")
|
|||
|
|
+ public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer {
|
|||
|
|
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);
|
|||
|
|
+@@ -305,6 +305,10 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+ @Override
|
|||
|
|
+ public void shutdown() {
|
|||
|
|
+ try {
|
|||
|
|
++ if (nettyServerConfig.isEnableShutdownGracefully() && isShuttingDown.compareAndSet(false, true)) {
|
|||
|
|
++ Thread.sleep(Duration.ofSeconds(nettyServerConfig.getShutdownWaitTimeSeconds()).toMillis());
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ this.timer.stop();
|
|||
|
|
+
|
|||
|
|
+ this.eventLoopGroupBoss.shutdownGracefully();
|
|||
|
|
+@@ -736,6 +740,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void shutdown() {
|
|||
|
|
++ isShuttingDown.set(true);
|
|||
|
|
+ if (this.serverChannel != null) {
|
|||
|
|
+ try {
|
|||
|
|
+ this.serverChannel.close().await(5, TimeUnit.SECONDS);
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java
|
|||
|
|
+index 59ef2c84f..756661f62 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java
|
|||
|
|
+@@ -38,6 +38,9 @@ public class NettyServerConfig implements Cloneable {
|
|||
|
|
+ private int serverSocketBacklog = NettySystemConfig.socketBacklog;
|
|||
|
|
+ private boolean serverPooledByteBufAllocatorEnable = true;
|
|||
|
|
+
|
|||
|
|
++ private boolean enableShutdownGracefully = false;
|
|||
|
|
++ private int shutdownWaitTimeSeconds = 30;
|
|||
|
|
++
|
|||
|
|
+ /**
|
|||
|
|
+ * make install
|
|||
|
|
+ *
|
|||
|
|
+@@ -171,4 +174,20 @@ public class NettyServerConfig implements Cloneable {
|
|||
|
|
+ public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) {
|
|||
|
|
+ this.writeBufferHighWaterMark = writeBufferHighWaterMark;
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public boolean isEnableShutdownGracefully() {
|
|||
|
|
++ return enableShutdownGracefully;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setEnableShutdownGracefully(boolean enableShutdownGracefully) {
|
|||
|
|
++ this.enableShutdownGracefully = enableShutdownGracefully;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public int getShutdownWaitTimeSeconds() {
|
|||
|
|
++ return shutdownWaitTimeSeconds;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public void setShutdownWaitTimeSeconds(int shutdownWaitTimeSeconds) {
|
|||
|
|
++ this.shutdownWaitTimeSeconds = shutdownWaitTimeSeconds;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java
|
|||
|
|
+index e81dadf2e..be945c48f 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java
|
|||
|
|
+@@ -99,6 +99,8 @@ public class ResponseCode extends RemotingSysResponseCode {
|
|||
|
|
+ public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004;
|
|||
|
|
+ public static final int RPC_TIME_OUT = -1006;
|
|||
|
|
+
|
|||
|
|
++ public static final int GO_AWAY = 1500;
|
|||
|
|
++
|
|||
|
|
+ /**
|
|||
|
|
+ * Controller response code
|
|||
|
|
+ */
|
|||
|
|
+diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
|
|||
|
|
+index e72e7bd53..1cc6b4f46 100644
|
|||
|
|
+--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
|
|||
|
|
++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
|
|||
|
|
+@@ -47,7 +47,6 @@ import static org.mockito.ArgumentMatchers.any;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyLong;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.anyString;
|
|||
|
|
+ import static org.mockito.ArgumentMatchers.eq;
|
|||
|
|
+-import static org.mockito.Mockito.doAnswer;
|
|||
|
|
+ import static org.mockito.Mockito.doReturn;
|
|||
|
|
+ import static org.mockito.Mockito.mock;
|
|||
|
|
+ import static org.mockito.Mockito.never;
|
|||
|
|
+@@ -74,13 +73,11 @@ public class NettyRemotingClientTest {
|
|||
|
|
+
|
|||
|
|
+ RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+- doAnswer(invocation -> {
|
|||
|
|
+- InvokeCallback callback = invocation.getArgument(3);
|
|||
|
|
+- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
|
|||
|
|
+- responseFuture.setResponseCommand(response);
|
|||
|
|
+- callback.operationSucceed(responseFuture.getResponseCommand());
|
|||
|
|
+- return null;
|
|||
|
|
+- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
|
|||
|
|
++ responseFuture.setResponseCommand(response);
|
|||
|
|
++ CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();
|
|||
|
|
++ future0.complete(responseFuture.getResponseCommand());
|
|||
|
|
++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
|
|||
|
|
+
|
|||
|
|
+ CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000);
|
|||
|
|
+ RemotingCommand actual = future.get();
|
|||
|
|
+@@ -93,11 +90,9 @@ public class NettyRemotingClientTest {
|
|||
|
|
+
|
|||
|
|
+ RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+- doAnswer(invocation -> {
|
|||
|
|
+- InvokeCallback callback = invocation.getArgument(3);
|
|||
|
|
+- callback.operationFail(new RemotingSendRequestException(null));
|
|||
|
|
+- return null;
|
|||
|
|
+- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
++ CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();
|
|||
|
|
++ future0.completeExceptionally(new RemotingSendRequestException(null));
|
|||
|
|
++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
|
|||
|
|
+
|
|||
|
|
+ CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000);
|
|||
|
|
+ Throwable thrown = catchThrowable(future::get);
|
|||
|
|
+@@ -110,11 +105,9 @@ public class NettyRemotingClientTest {
|
|||
|
|
+
|
|||
|
|
+ RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+ response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+- doAnswer(invocation -> {
|
|||
|
|
+- InvokeCallback callback = invocation.getArgument(3);
|
|||
|
|
+- callback.operationFail(new RemotingTimeoutException(""));
|
|||
|
|
+- return null;
|
|||
|
|
+- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
++ CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();
|
|||
|
|
++ future0.completeExceptionally(new RemotingTimeoutException(""));
|
|||
|
|
++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
|
|||
|
|
+
|
|||
|
|
+ CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000);
|
|||
|
|
+ Throwable thrown = catchThrowable(future::get);
|
|||
|
|
+@@ -125,13 +118,9 @@ public class NettyRemotingClientTest {
|
|||
|
|
+ public void testRemotingException() throws Exception {
|
|||
|
|
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
|
|||
|
|
+
|
|||
|
|
+- RemotingCommand response = RemotingCommand.createResponseCommand(null);
|
|||
|
|
+- response.setCode(ResponseCode.SUCCESS);
|
|||
|
|
+- doAnswer(invocation -> {
|
|||
|
|
+- InvokeCallback callback = invocation.getArgument(3);
|
|||
|
|
+- callback.operationFail(new RemotingException(null));
|
|||
|
|
+- return null;
|
|||
|
|
+- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
|
|||
|
|
++ CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();
|
|||
|
|
++ future0.completeExceptionally(new RemotingException(""));
|
|||
|
|
++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
|
|||
|
|
+
|
|||
|
|
+ CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000);
|
|||
|
|
+ Throwable thrown = catchThrowable(future::get);
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/patch021-backport-some-enhancements.patch b/patch021-backport-some-enhancements.patch
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..839b65fea
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/patch021-backport-some-enhancements.patch
|
|||
|
|
@@ -0,0 +1,344 @@
|
|||
|
|
+From dc3f22ffe9eb83ace991b68921076093c7c0da5f Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: LetLetMe <43874697+LetLetMe@users.noreply.github.com>
|
|||
|
|
+Date: Tue, 10 Oct 2023 17:39:23 +0800
|
|||
|
|
+Subject: [PATCH 1/6] add getter for class Message ,fix json serialize bug
|
|||
|
|
+ (#7439)
|
|||
|
|
+
|
|||
|
|
+Co-authored-by: LetLetMe <allen.hyt@alibaba-inc.com>
|
|||
|
|
+---
|
|||
|
|
+ .../rocketmq/common/message/Message.java | 24 ++++++++++++++++++-
|
|||
|
|
+ 1 file changed, 23 insertions(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/message/Message.java b/common/src/main/java/org/apache/rocketmq/common/message/Message.java
|
|||
|
|
+index e02b526a1..c7997c473 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/message/Message.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/message/Message.java
|
|||
|
|
+@@ -218,14 +218,36 @@ public class Message implements Serializable {
|
|||
|
|
+ public void setDelayTimeSec(long sec) {
|
|||
|
|
+ this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC, String.valueOf(sec));
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public long getDelayTimeSec() {
|
|||
|
|
++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC);
|
|||
|
|
++ if (t != null) {
|
|||
|
|
++ return Long.parseLong(t);
|
|||
|
|
++ }
|
|||
|
|
++ return 0;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public void setDelayTimeMs(long timeMs) {
|
|||
|
|
+ this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_MS, String.valueOf(timeMs));
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ public long getDelayTimeMs() {
|
|||
|
|
++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS);
|
|||
|
|
++ if (t != null) {
|
|||
|
|
++ return Long.parseLong(t);
|
|||
|
|
++ }
|
|||
|
|
++ return 0;
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
+ public void setDeliverTimeMs(long timeMs) {
|
|||
|
|
+ this.putProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(timeMs));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long getDeliverTimeMs() {
|
|||
|
|
+- return Long.parseLong(this.getUserProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS));
|
|||
|
|
++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS);
|
|||
|
|
++ if (t != null) {
|
|||
|
|
++ return Long.parseLong(t);
|
|||
|
|
++ }
|
|||
|
|
++ return 0;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 7e4879a3bc120d6289aabc8354a2811f349ac8a6 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: fujian-zfj <2573259572@qq.com>
|
|||
|
|
+Date: Wed, 11 Oct 2023 14:45:07 +0800
|
|||
|
|
+Subject: [PATCH 2/6] [ISSUE #7441] Fix log "Init the confirmOffset" keep
|
|||
|
|
+ printing error in controller mode (#7442)
|
|||
|
|
+
|
|||
|
|
+* typo int readme[ecosystem]
|
|||
|
|
+
|
|||
|
|
+* fix keep printing log problem
|
|||
|
|
+---
|
|||
|
|
+ store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 2 +-
|
|||
|
|
+ 1 file changed, 1 insertion(+), 1 deletion(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
|
|||
|
|
+index 456bf2b86..f98e9a284 100644
|
|||
|
|
+--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
|
|||
|
|
++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
|
|||
|
|
+@@ -580,7 +580,7 @@ public class CommitLog implements Swappable {
|
|||
|
|
+ return this.defaultMessageStore.getMaxPhyOffset();
|
|||
|
|
+ }
|
|||
|
|
+ // First time it will compute the confirmOffset.
|
|||
|
|
+- if (this.confirmOffset <= 0) {
|
|||
|
|
++ if (this.confirmOffset < 0) {
|
|||
|
|
+ setConfirmOffset(((AutoSwitchHAService) this.defaultMessageStore.getHaService()).computeConfirmOffset());
|
|||
|
|
+ log.info("Init the confirmOffset to {}.", this.confirmOffset);
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 5d492c338258d07613103e6ae16df4c6fa5b3838 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: rongtong <jinrongtong5@163.com>
|
|||
|
|
+Date: Fri, 13 Oct 2023 11:23:30 +0800
|
|||
|
|
+Subject: [PATCH 3/6] [ISSUE #7444] Fix testCalculateFileSizeInPath test can
|
|||
|
|
+ not rerun in same environment (#7445)
|
|||
|
|
+
|
|||
|
|
+* Fix testCalculateFileSizeInPath test can not rerun in same environment
|
|||
|
|
+
|
|||
|
|
+* Ensure that files are always deleted
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/common/UtilAllTest.java | 83 +++++++++++--------
|
|||
|
|
+ 1 file changed, 48 insertions(+), 35 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java
|
|||
|
|
+index f568a65f4..a0653d7fc 100644
|
|||
|
|
+--- a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java
|
|||
|
|
++++ b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java
|
|||
|
|
+@@ -238,41 +238,54 @@ public class UtilAllTest {
|
|||
|
|
+ */
|
|||
|
|
+ String basePath = System.getProperty("java.io.tmpdir") + File.separator + "testCalculateFileSizeInPath";
|
|||
|
|
+ File baseFile = new File(basePath);
|
|||
|
|
+- // test empty path
|
|||
|
|
+- assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile));
|
|||
|
|
+-
|
|||
|
|
+- // create baseDir
|
|||
|
|
+- assertTrue(baseFile.mkdirs());
|
|||
|
|
+-
|
|||
|
|
+- File file0 = new File(baseFile, "file_0");
|
|||
|
|
+- assertTrue(file0.createNewFile());
|
|||
|
|
+- writeFixedBytesToFile(file0, 1313);
|
|||
|
|
+-
|
|||
|
|
+- assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile));
|
|||
|
|
+-
|
|||
|
|
+- // build a file tree like above
|
|||
|
|
+- File dir1 = new File(baseFile, "dir_1");
|
|||
|
|
+- dir1.mkdirs();
|
|||
|
|
+- File file10 = new File(dir1, "file_1_0");
|
|||
|
|
+- File file11 = new File(dir1, "file_1_1");
|
|||
|
|
+- File dir12 = new File(dir1, "dir_1_2");
|
|||
|
|
+- dir12.mkdirs();
|
|||
|
|
+- File file120 = new File(dir12, "file_1_2_0");
|
|||
|
|
+- File dir2 = new File(baseFile, "dir_2");
|
|||
|
|
+- dir2.mkdirs();
|
|||
|
|
+-
|
|||
|
|
+- // write all file with 1313 bytes data
|
|||
|
|
+- assertTrue(file10.createNewFile());
|
|||
|
|
+- writeFixedBytesToFile(file10, 1313);
|
|||
|
|
+- assertTrue(file11.createNewFile());
|
|||
|
|
+- writeFixedBytesToFile(file11, 1313);
|
|||
|
|
+- assertTrue(file120.createNewFile());
|
|||
|
|
+- writeFixedBytesToFile(file120, 1313);
|
|||
|
|
+-
|
|||
|
|
+- assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile));
|
|||
|
|
+-
|
|||
|
|
+- // clear all file
|
|||
|
|
+- baseFile.deleteOnExit();
|
|||
|
|
++ try {
|
|||
|
|
++ // test empty path
|
|||
|
|
++ assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile));
|
|||
|
|
++
|
|||
|
|
++ // create baseDir
|
|||
|
|
++ assertTrue(baseFile.mkdirs());
|
|||
|
|
++
|
|||
|
|
++ File file0 = new File(baseFile, "file_0");
|
|||
|
|
++ assertTrue(file0.createNewFile());
|
|||
|
|
++ writeFixedBytesToFile(file0, 1313);
|
|||
|
|
++
|
|||
|
|
++ assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile));
|
|||
|
|
++
|
|||
|
|
++ // build a file tree like above
|
|||
|
|
++ File dir1 = new File(baseFile, "dir_1");
|
|||
|
|
++ dir1.mkdirs();
|
|||
|
|
++ File file10 = new File(dir1, "file_1_0");
|
|||
|
|
++ File file11 = new File(dir1, "file_1_1");
|
|||
|
|
++ File dir12 = new File(dir1, "dir_1_2");
|
|||
|
|
++ dir12.mkdirs();
|
|||
|
|
++ File file120 = new File(dir12, "file_1_2_0");
|
|||
|
|
++ File dir2 = new File(baseFile, "dir_2");
|
|||
|
|
++ dir2.mkdirs();
|
|||
|
|
++
|
|||
|
|
++ // write all file with 1313 bytes data
|
|||
|
|
++ assertTrue(file10.createNewFile());
|
|||
|
|
++ writeFixedBytesToFile(file10, 1313);
|
|||
|
|
++ assertTrue(file11.createNewFile());
|
|||
|
|
++ writeFixedBytesToFile(file11, 1313);
|
|||
|
|
++ assertTrue(file120.createNewFile());
|
|||
|
|
++ writeFixedBytesToFile(file120, 1313);
|
|||
|
|
++
|
|||
|
|
++ assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile));
|
|||
|
|
++ } finally {
|
|||
|
|
++ deleteFolder(baseFile);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++
|
|||
|
|
++ public static void deleteFolder(File folder) {
|
|||
|
|
++ if (folder.isDirectory()) {
|
|||
|
|
++ File[] files = folder.listFiles();
|
|||
|
|
++ if (files != null) {
|
|||
|
|
++ for (File file : files) {
|
|||
|
|
++ deleteFolder(file);
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ }
|
|||
|
|
++ folder.delete();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void writeFixedBytesToFile(File file, int size) throws Exception {
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 28427d40129e3aa0c6f951535617e5cac0a8211b Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: Lei Sun <yuncun.sl@alibaba-inc.com>
|
|||
|
|
+Date: Fri, 13 Oct 2023 13:42:27 +0800
|
|||
|
|
+Subject: [PATCH 4/6] [ISSUE #7425] Add RoccketmqControllerConsole log to fix
|
|||
|
|
+ bug (#7458)
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../org/apache/rocketmq/common/constant/LoggerName.java | 1 +
|
|||
|
|
+ .../org/apache/rocketmq/controller/ControllerStartup.java | 7 ++++---
|
|||
|
|
+ controller/src/main/resources/rmq.controller.logback.xml | 4 ++++
|
|||
|
|
+ 3 files changed, 9 insertions(+), 3 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
|
|||
|
|
+index cb04b00b3..61310893f 100644
|
|||
|
|
+--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
|
|||
|
|
++++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
|
|||
|
|
+@@ -21,6 +21,7 @@ public class LoggerName {
|
|||
|
|
+ public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv";
|
|||
|
|
+ public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole";
|
|||
|
|
+ public static final String CONTROLLER_LOGGER_NAME = "RocketmqController";
|
|||
|
|
++ public static final String CONTROLLER_CONSOLE_NAME = "RocketmqControllerConsole";
|
|||
|
|
+ public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark";
|
|||
|
|
+ public static final String BROKER_LOGGER_NAME = "RocketmqBroker";
|
|||
|
|
+ public static final String BROKER_CONSOLE_NAME = "RocketmqConsole";
|
|||
|
|
+diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java
|
|||
|
|
+index 401720d05..9e96a704d 100644
|
|||
|
|
+--- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java
|
|||
|
|
++++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java
|
|||
|
|
+@@ -94,9 +94,10 @@ public class ControllerStartup {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (commandLine.hasOption('p')) {
|
|||
|
|
+- MixAll.printObjectProperties(null, controllerConfig);
|
|||
|
|
+- MixAll.printObjectProperties(null, nettyServerConfig);
|
|||
|
|
+- MixAll.printObjectProperties(null, nettyClientConfig);
|
|||
|
|
++ Logger console = LoggerFactory.getLogger(LoggerName.CONTROLLER_CONSOLE_NAME);
|
|||
|
|
++ MixAll.printObjectProperties(console, controllerConfig);
|
|||
|
|
++ MixAll.printObjectProperties(console, nettyServerConfig);
|
|||
|
|
++ MixAll.printObjectProperties(console, nettyClientConfig);
|
|||
|
|
+ System.exit(0);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+diff --git a/controller/src/main/resources/rmq.controller.logback.xml b/controller/src/main/resources/rmq.controller.logback.xml
|
|||
|
|
+index bb158213a..18083e8f9 100644
|
|||
|
|
+--- a/controller/src/main/resources/rmq.controller.logback.xml
|
|||
|
|
++++ b/controller/src/main/resources/rmq.controller.logback.xml
|
|||
|
|
+@@ -116,6 +116,10 @@
|
|||
|
|
+ <appender-ref ref="RocketmqControllerAppender"/>
|
|||
|
|
+ </logger>
|
|||
|
|
+
|
|||
|
|
++ <logger name="RocketmqControllerConsole" additivity="false" level="INFO">
|
|||
|
|
++ <appender-ref ref="STDOUT"/>
|
|||
|
|
++ </logger>
|
|||
|
|
++
|
|||
|
|
+ <logger name="RocketmqCommon" additivity="false" level="INFO">
|
|||
|
|
+ <appender-ref ref="RocketmqControllerAppender"/>
|
|||
|
|
+ </logger>
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From dc62d7f2e1ce4f99364599f8e23d65fd88eb1cd4 Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: LetLetMe <43874697+LetLetMe@users.noreply.github.com>
|
|||
|
|
+Date: Fri, 13 Oct 2023 13:45:48 +0800
|
|||
|
|
+Subject: [PATCH 5/6] [ISSUE #7451] Override toString for
|
|||
|
|
+ TopicConfigAndQueueMapping
|
|||
|
|
+
|
|||
|
|
+---
|
|||
|
|
+ .../statictopic/TopicConfigAndQueueMapping.java | 10 ++++++++++
|
|||
|
|
+ 1 file changed, 10 insertions(+)
|
|||
|
|
+
|
|||
|
|
+diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java
|
|||
|
|
+index c937fec23..d13692735 100644
|
|||
|
|
+--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java
|
|||
|
|
++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java
|
|||
|
|
+@@ -16,6 +16,7 @@
|
|||
|
|
+ */
|
|||
|
|
+ package org.apache.rocketmq.remoting.protocol.statictopic;
|
|||
|
|
+
|
|||
|
|
++import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+ import org.apache.commons.lang3.builder.EqualsBuilder;
|
|||
|
|
+ import org.apache.commons.lang3.builder.HashCodeBuilder;
|
|||
|
|
+ import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
+@@ -60,4 +61,13 @@ public class TopicConfigAndQueueMapping extends TopicConfig {
|
|||
|
|
+ .append(mappingDetail)
|
|||
|
|
+ .toHashCode();
|
|||
|
|
+ }
|
|||
|
|
++
|
|||
|
|
++ @Override
|
|||
|
|
++ public String toString() {
|
|||
|
|
++ String string = super.toString();
|
|||
|
|
++ if (StringUtils.isNotBlank(string)) {
|
|||
|
|
++ string = string.substring(0, string.length() - 1) + ", mappingDetail=" + mappingDetail + "]";
|
|||
|
|
++ }
|
|||
|
|
++ return string;
|
|||
|
|
++ }
|
|||
|
|
+ }
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+From 2113fa371b9c2bf7c512f8ad234e51c616f1362c Mon Sep 17 00:00:00 2001
|
|||
|
|
+From: guyinyou <36399867+guyinyou@users.noreply.github.com>
|
|||
|
|
+Date: Fri, 13 Oct 2023 13:47:09 +0800
|
|||
|
|
+Subject: [PATCH 6/6] [ISSUE #7453] Fix the problem in constructing the
|
|||
|
|
+ GetMessageResult (#7456)
|
|||
|
|
+
|
|||
|
|
+* Fix the problem in constructing the GetMessageResult
|
|||
|
|
+
|
|||
|
|
+* Optimize the initialization size of GetMessageResult
|
|||
|
|
+---
|
|||
|
|
+ .../apache/rocketmq/broker/processor/PeekMessageProcessor.java | 3 +--
|
|||
|
|
+ .../apache/rocketmq/broker/processor/PopMessageProcessor.java | 3 +--
|
|||
|
|
+ 2 files changed, 2 insertions(+), 4 deletions(-)
|
|||
|
|
+
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
|
|||
|
|
+index a8358c4ff..e1e0e13e5 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
|
|||
|
|
+@@ -129,8 +129,7 @@ public class PeekMessageProcessor implements NettyRequestProcessor {
|
|||
|
|
+ }
|
|||
|
|
+ int randomQ = random.nextInt(100);
|
|||
|
|
+ int reviveQid = randomQ % this.brokerController.getBrokerConfig().getReviveQueueNum();
|
|||
|
|
+- int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg();
|
|||
|
|
+- GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg);
|
|||
|
|
++ GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums());
|
|||
|
|
+ boolean needRetry = randomQ % 5 == 0;
|
|||
|
|
+ long popTime = System.currentTimeMillis();
|
|||
|
|
+ long restNum = 0;
|
|||
|
|
+diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
|
|||
|
|
+index 441f7de08..0d9bdf143 100644
|
|||
|
|
+--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
|
|||
|
|
++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
|
|||
|
|
+@@ -347,8 +347,7 @@ public class PopMessageProcessor implements NettyRequestProcessor {
|
|||
|
|
+ reviveQid = (int) Math.abs(ckMessageNumber.getAndIncrement() % this.brokerController.getBrokerConfig().getReviveQueueNum());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+- int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg();
|
|||
|
|
+- GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg);
|
|||
|
|
++ GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums());
|
|||
|
|
+ ExpressionMessageFilter finalMessageFilter = messageFilter;
|
|||
|
|
+ StringBuilder finalOrderCountInfo = orderCountInfo;
|
|||
|
|
+
|
|||
|
|
+--
|
|||
|
|
+2.32.0.windows.2
|
|||
|
|
+
|
|||
|
|
diff --git a/pom.xml b/pom.xml
|
|||
|
|
index 4202d4095..a3f7c2270 100644
|
|||
|
|
--- a/pom.xml
|
|||
|
|
+++ b/pom.xml
|
|||
|
|
@@ -137,7 +137,7 @@
|
|||
|
|
<opentelemetry-exporter-prometheus.version>1.29.0-alpha</opentelemetry-exporter-prometheus.version>
|
|||
|
|
<jul-to-slf4j.version>2.0.6</jul-to-slf4j.version>
|
|||
|
|
<s3.version>2.20.29</s3.version>
|
|||
|
|
- <rocksdb.version>1.0.3</rocksdb.version>
|
|||
|
|
+ <rocksdb.version>1.0.2</rocksdb.version>
|
|||
|
|
<jackson-databind.version>2.13.4.2</jackson-databind.version>
|
|||
|
|
|
|||
|
|
<!-- Test dependencies -->
|
|||
|
|
@@ -713,7 +713,7 @@
|
|||
|
|
<version>${slf4j-api.version}</version>
|
|||
|
|
</dependency>
|
|||
|
|
<dependency>
|
|||
|
|
- <groupId>io.github.aliyunmq</groupId>
|
|||
|
|
+ <groupId>org.apache.rocketmq</groupId>
|
|||
|
|
<artifactId>rocketmq-rocksdb</artifactId>
|
|||
|
|
<version>${rocksdb.version}</version>
|
|||
|
|
</dependency>
|
|||
|
|
diff --git a/store/pom.xml b/store/pom.xml
|
|||
|
|
index e979030e8..e1e616123 100644
|
|||
|
|
--- a/store/pom.xml
|
|||
|
|
+++ b/store/pom.xml
|
|||
|
|
@@ -58,6 +58,10 @@
|
|||
|
|
<groupId>com.google.guava</groupId>
|
|||
|
|
<artifactId>guava</artifactId>
|
|||
|
|
</dependency>
|
|||
|
|
+ <dependency>
|
|||
|
|
+ <groupId>commons-io</groupId>
|
|||
|
|
+ <artifactId>commons-io</artifactId>
|
|||
|
|
+ </dependency>
|
|||
|
|
<!-- Required by DLedger -->
|
|||
|
|
<dependency>
|
|||
|
|
<groupId>org.slf4j</groupId>
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
|
|||
|
|
index f98e9a284..93102799b 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
|
|||
|
|
@@ -60,6 +60,8 @@ import org.apache.rocketmq.store.ha.HAService;
|
|||
|
|
import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;
|
|||
|
|
import org.apache.rocketmq.store.logfile.MappedFile;
|
|||
|
|
import org.apache.rocketmq.store.util.LibC;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+
|
|||
|
|
import sun.nio.ch.DirectBuffer;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
@@ -299,8 +301,9 @@ public class CommitLog implements Swappable {
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* When the normal exit, data recovery, all memory data have been flush
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
*/
|
|||
|
|
- public void recoverNormally(long maxPhyOffsetOfConsumeQueue) {
|
|||
|
|
+ public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException {
|
|||
|
|
boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();
|
|||
|
|
boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable();
|
|||
|
|
final List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();
|
|||
|
|
@@ -369,21 +372,22 @@ public class CommitLog implements Swappable {
|
|||
|
|
this.setConfirmOffset(lastValidMsgPhyOffset);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- this.mappedFileQueue.setFlushedWhere(processOffset);
|
|||
|
|
- this.mappedFileQueue.setCommittedWhere(processOffset);
|
|||
|
|
- this.mappedFileQueue.truncateDirtyFiles(processOffset);
|
|||
|
|
-
|
|||
|
|
// Clear ConsumeQueue redundant data
|
|||
|
|
if (maxPhyOffsetOfConsumeQueue >= processOffset) {
|
|||
|
|
log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset);
|
|||
|
|
this.defaultMessageStore.truncateDirtyLogicFiles(processOffset);
|
|||
|
|
}
|
|||
|
|
+
|
|||
|
|
+ this.mappedFileQueue.setFlushedWhere(processOffset);
|
|||
|
|
+ this.mappedFileQueue.setCommittedWhere(processOffset);
|
|||
|
|
+ this.mappedFileQueue.truncateDirtyFiles(processOffset);
|
|||
|
|
} else {
|
|||
|
|
// Commitlog case files are deleted
|
|||
|
|
log.warn("The commitlog files are deleted, and delete the consume queue files");
|
|||
|
|
this.mappedFileQueue.setFlushedWhere(0);
|
|||
|
|
this.mappedFileQueue.setCommittedWhere(0);
|
|||
|
|
- this.defaultMessageStore.destroyLogics();
|
|||
|
|
+ this.defaultMessageStore.getQueueStore().destroy();
|
|||
|
|
+ this.defaultMessageStore.getQueueStore().loadAfterDestroy();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -626,8 +630,10 @@ public class CommitLog implements Swappable {
|
|||
|
|
return -1;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- @Deprecated
|
|||
|
|
- public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) {
|
|||
|
|
+ /**
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException {
|
|||
|
|
// recover by the minimum time stamp
|
|||
|
|
boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();
|
|||
|
|
boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable();
|
|||
|
|
@@ -705,6 +711,9 @@ public class CommitLog implements Swappable {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ // only for rocksdb mode
|
|||
|
|
+ this.getMessageStore().finishCommitLogDispatch();
|
|||
|
|
+
|
|||
|
|
processOffset += mappedFileOffset;
|
|||
|
|
if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {
|
|||
|
|
if (this.defaultMessageStore.getConfirmOffset() < this.defaultMessageStore.getMinPhyOffset()) {
|
|||
|
|
@@ -717,22 +726,24 @@ public class CommitLog implements Swappable {
|
|||
|
|
} else {
|
|||
|
|
this.setConfirmOffset(lastValidMsgPhyOffset);
|
|||
|
|
}
|
|||
|
|
- this.mappedFileQueue.setFlushedWhere(processOffset);
|
|||
|
|
- this.mappedFileQueue.setCommittedWhere(processOffset);
|
|||
|
|
- this.mappedFileQueue.truncateDirtyFiles(processOffset);
|
|||
|
|
|
|||
|
|
// Clear ConsumeQueue redundant data
|
|||
|
|
if (maxPhyOffsetOfConsumeQueue >= processOffset) {
|
|||
|
|
log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset);
|
|||
|
|
this.defaultMessageStore.truncateDirtyLogicFiles(processOffset);
|
|||
|
|
}
|
|||
|
|
+
|
|||
|
|
+ this.mappedFileQueue.setFlushedWhere(processOffset);
|
|||
|
|
+ this.mappedFileQueue.setCommittedWhere(processOffset);
|
|||
|
|
+ this.mappedFileQueue.truncateDirtyFiles(processOffset);
|
|||
|
|
}
|
|||
|
|
// Commitlog case files are deleted
|
|||
|
|
else {
|
|||
|
|
log.warn("The commitlog files are deleted, and delete the consume queue files");
|
|||
|
|
this.mappedFileQueue.setFlushedWhere(0);
|
|||
|
|
this.mappedFileQueue.setCommittedWhere(0);
|
|||
|
|
- this.defaultMessageStore.destroyLogics();
|
|||
|
|
+ this.defaultMessageStore.getQueueStore().destroy();
|
|||
|
|
+ this.defaultMessageStore.getQueueStore().loadAfterDestroy();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -755,7 +766,7 @@ public class CommitLog implements Swappable {
|
|||
|
|
this.getMessageStore().onCommitLogAppend(msg, result, commitLogFile);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) {
|
|||
|
|
+ private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) throws RocksDBException {
|
|||
|
|
ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();
|
|||
|
|
|
|||
|
|
int magicCode = byteBuffer.getInt(MessageDecoder.MESSAGE_MAGIC_CODE_POSITION);
|
|||
|
|
@@ -763,28 +774,37 @@ public class CommitLog implements Swappable {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION);
|
|||
|
|
- int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;
|
|||
|
|
- int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornhostLength;
|
|||
|
|
- long storeTimestamp = byteBuffer.getLong(msgStoreTimePos);
|
|||
|
|
- if (0 == storeTimestamp) {
|
|||
|
|
- return false;
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable()
|
|||
|
|
- && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) {
|
|||
|
|
- if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) {
|
|||
|
|
- log.info("find check timestamp, {} {}",
|
|||
|
|
- storeTimestamp,
|
|||
|
|
- UtilAll.timeMillisToHumanString(storeTimestamp));
|
|||
|
|
+ if (this.defaultMessageStore.getMessageStoreConfig().isEnableRocksDBStore()) {
|
|||
|
|
+ final long maxPhyOffsetInConsumeQueue = this.defaultMessageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue();
|
|||
|
|
+ long phyOffset = byteBuffer.getLong(MessageDecoder.MESSAGE_PHYSIC_OFFSET_POSITION);
|
|||
|
|
+ if (phyOffset <= maxPhyOffsetInConsumeQueue) {
|
|||
|
|
+ log.info("find check. beginPhyOffset: {}, maxPhyOffsetInConsumeQueue: {}", phyOffset, maxPhyOffsetInConsumeQueue);
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
- if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) {
|
|||
|
|
- log.info("find check timestamp, {} {}",
|
|||
|
|
- storeTimestamp,
|
|||
|
|
- UtilAll.timeMillisToHumanString(storeTimestamp));
|
|||
|
|
- return true;
|
|||
|
|
+ int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION);
|
|||
|
|
+ int bornHostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;
|
|||
|
|
+ int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornHostLength;
|
|||
|
|
+ long storeTimestamp = byteBuffer.getLong(msgStoreTimePos);
|
|||
|
|
+ if (0 == storeTimestamp) {
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable()
|
|||
|
|
+ && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) {
|
|||
|
|
+ if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) {
|
|||
|
|
+ log.info("find check timestamp, {} {}",
|
|||
|
|
+ storeTimestamp,
|
|||
|
|
+ UtilAll.timeMillisToHumanString(storeTimestamp));
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+ } else {
|
|||
|
|
+ if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) {
|
|||
|
|
+ log.info("find check timestamp, {} {}",
|
|||
|
|
+ storeTimestamp,
|
|||
|
|
+ UtilAll.timeMillisToHumanString(storeTimestamp));
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -958,8 +978,6 @@ public class CommitLog implements Swappable {
|
|||
|
|
beginTimeInLock = 0;
|
|||
|
|
return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
|
|||
|
|
case UNKNOWN_ERROR:
|
|||
|
|
- beginTimeInLock = 0;
|
|||
|
|
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
|
|||
|
|
default:
|
|||
|
|
beginTimeInLock = 0;
|
|||
|
|
return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
|
|||
|
|
@@ -974,6 +992,8 @@ public class CommitLog implements Swappable {
|
|||
|
|
if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) {
|
|||
|
|
this.defaultMessageStore.increaseOffset(msg, getMessageNum(msg));
|
|||
|
|
}
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
|
|||
|
|
} finally {
|
|||
|
|
topicQueueLock.unlock(topicQueueKey);
|
|||
|
|
}
|
|||
|
|
@@ -997,7 +1017,7 @@ public class CommitLog implements Swappable {
|
|||
|
|
|
|||
|
|
public CompletableFuture<PutMessageResult> asyncPutMessages(final MessageExtBatch messageExtBatch) {
|
|||
|
|
messageExtBatch.setStoreTimestamp(System.currentTimeMillis());
|
|||
|
|
- AppendMessageResult result;
|
|||
|
|
+ AppendMessageResult result = null;
|
|||
|
|
|
|||
|
|
StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();
|
|||
|
|
|
|||
|
|
@@ -1133,7 +1153,9 @@ public class CommitLog implements Swappable {
|
|||
|
|
if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) {
|
|||
|
|
this.defaultMessageStore.increaseOffset(messageExtBatch, (short) putMessageContext.getBatchSize());
|
|||
|
|
}
|
|||
|
|
- } finally {
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
|
|||
|
|
+ } finally {
|
|||
|
|
topicQueueLock.unlock(topicQueueKey);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java
|
|||
|
|
index 9d6fa6ad9..f3a7b7c5c 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java
|
|||
|
|
@@ -17,6 +17,8 @@
|
|||
|
|
|
|||
|
|
package org.apache.rocketmq.store;
|
|||
|
|
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+
|
|||
|
|
/**
|
|||
|
|
* Dispatcher of commit log.
|
|||
|
|
*/
|
|||
|
|
@@ -25,6 +27,7 @@ public interface CommitLogDispatcher {
|
|||
|
|
/**
|
|||
|
|
* Dispatch messages from store to build consume queues, indexes, and filter data
|
|||
|
|
* @param request dispatch message request
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
*/
|
|||
|
|
- void dispatch(final DispatchRequest request);
|
|||
|
|
+ void dispatch(final DispatchRequest request) throws RocksDBException;
|
|||
|
|
}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
|
|||
|
|
index 56bee2af3..623509c8b 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
|
|||
|
|
@@ -24,13 +24,12 @@ import java.util.Map;
|
|||
|
|
import org.apache.commons.lang3.StringUtils;
|
|||
|
|
import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+import org.apache.rocketmq.common.Pair;
|
|||
|
|
import org.apache.rocketmq.common.attribute.CQType;
|
|||
|
|
import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageAccessor;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageConst;
|
|||
|
|
-import org.apache.rocketmq.common.message.MessageDecoder;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
-import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
import org.apache.rocketmq.store.config.BrokerRole;
|
|||
|
|
@@ -39,9 +38,9 @@ import org.apache.rocketmq.store.logfile.MappedFile;
|
|||
|
|
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
|
|||
|
|
import org.apache.rocketmq.store.queue.CqUnit;
|
|||
|
|
import org.apache.rocketmq.store.queue.FileQueueLifeCycle;
|
|||
|
|
+import org.apache.rocketmq.store.queue.MultiDispatch;
|
|||
|
|
import org.apache.rocketmq.store.queue.QueueOffsetOperator;
|
|||
|
|
import org.apache.rocketmq.store.queue.ReferredIterator;
|
|||
|
|
-import org.apache.rocketmq.store.timer.TimerMessageStore;
|
|||
|
|
|
|||
|
|
public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
|
|||
|
|
@@ -703,7 +702,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp());
|
|||
|
|
}
|
|||
|
|
this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(request.getStoreTimestamp());
|
|||
|
|
- if (checkMultiDispatchQueue(request)) {
|
|||
|
|
+ if (MultiDispatch.checkMultiDispatchQueue(this.messageStore.getMessageStoreConfig(), request)) {
|
|||
|
|
multiDispatchLmqQueue(request, maxRetries);
|
|||
|
|
}
|
|||
|
|
return;
|
|||
|
|
@@ -725,25 +724,6 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
this.messageStore.getRunningFlags().makeLogicsQueueError();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private boolean checkMultiDispatchQueue(DispatchRequest dispatchRequest) {
|
|||
|
|
- if (!this.messageStore.getMessageStoreConfig().isEnableMultiDispatch()
|
|||
|
|
- || dispatchRequest.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)
|
|||
|
|
- || dispatchRequest.getTopic().equals(TimerMessageStore.TIMER_TOPIC)
|
|||
|
|
- || dispatchRequest.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC)) {
|
|||
|
|
- return false;
|
|||
|
|
- }
|
|||
|
|
- Map<String, String> prop = dispatchRequest.getPropertiesMap();
|
|||
|
|
- if (prop == null || prop.isEmpty()) {
|
|||
|
|
- return false;
|
|||
|
|
- }
|
|||
|
|
- String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
|
|||
|
|
- String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);
|
|||
|
|
- if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) {
|
|||
|
|
- return false;
|
|||
|
|
- }
|
|||
|
|
- return true;
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
private void multiDispatchLmqQueue(DispatchRequest request, int maxRetries) {
|
|||
|
|
Map<String, String> prop = request.getPropertiesMap();
|
|||
|
|
String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
|
|||
|
|
@@ -765,9 +745,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
queueId = 0;
|
|||
|
|
}
|
|||
|
|
doDispatchLmqQueue(request, maxRetries, queueName, queueOffset, queueId);
|
|||
|
|
-
|
|||
|
|
}
|
|||
|
|
- return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private void doDispatchLmqQueue(DispatchRequest request, int maxRetries, String queueName, long queueOffset,
|
|||
|
|
@@ -802,7 +780,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
|
|||
|
|
// Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28),
|
|||
|
|
// light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue.
|
|||
|
|
- if (!isNeedHandleMultiDispatch(msg)) {
|
|||
|
|
+ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) {
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
|
|||
|
|
@@ -812,14 +790,14 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
|
|||
|
|
Long[] queueOffsets = new Long[queues.length];
|
|||
|
|
for (int i = 0; i < queues.length; i++) {
|
|||
|
|
- String key = queueKey(queues[i], msg);
|
|||
|
|
- if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) {
|
|||
|
|
+ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) {
|
|||
|
|
+ String key = MultiDispatch.lmqQueueKey(queues[i]);
|
|||
|
|
queueOffsets[i] = queueOffsetOperator.getLmqOffset(key);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET,
|
|||
|
|
StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER));
|
|||
|
|
- removeWaitStorePropertyString(msg);
|
|||
|
|
+ msg.removeWaitStorePropertyString();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
@@ -830,7 +808,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
|
|||
|
|
// Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28),
|
|||
|
|
// light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue.
|
|||
|
|
- if (!isNeedHandleMultiDispatch(msg)) {
|
|||
|
|
+ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) {
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
|
|||
|
|
@@ -839,45 +817,13 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
}
|
|||
|
|
String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
|
|||
|
|
for (int i = 0; i < queues.length; i++) {
|
|||
|
|
- String key = queueKey(queues[i], msg);
|
|||
|
|
- if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) {
|
|||
|
|
+ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) {
|
|||
|
|
+ String key = MultiDispatch.lmqQueueKey(queues[i]);
|
|||
|
|
queueOffsetOperator.increaseLmqOffset(key, (short) 1);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public boolean isNeedHandleMultiDispatch(MessageExtBrokerInner msg) {
|
|||
|
|
- return messageStore.getMessageStoreConfig().isEnableMultiDispatch()
|
|||
|
|
- && !msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)
|
|||
|
|
- && !msg.getTopic().equals(TimerMessageStore.TIMER_TOPIC)
|
|||
|
|
- && !msg.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public String queueKey(String queueName, MessageExtBrokerInner msgInner) {
|
|||
|
|
- StringBuilder keyBuilder = new StringBuilder();
|
|||
|
|
- keyBuilder.append(queueName);
|
|||
|
|
- keyBuilder.append('-');
|
|||
|
|
- int queueId = msgInner.getQueueId();
|
|||
|
|
- if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {
|
|||
|
|
- queueId = 0;
|
|||
|
|
- }
|
|||
|
|
- keyBuilder.append(queueId);
|
|||
|
|
- return keyBuilder.toString();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- private void removeWaitStorePropertyString(MessageExtBrokerInner msgInner) {
|
|||
|
|
- if (msgInner.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) {
|
|||
|
|
- // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message.
|
|||
|
|
- // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it.
|
|||
|
|
- String waitStoreMsgOKValue = msgInner.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK);
|
|||
|
|
- msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
|
|||
|
|
- // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later
|
|||
|
|
- msgInner.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue);
|
|||
|
|
- } else {
|
|||
|
|
- msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode,
|
|||
|
|
final long cqOffset) {
|
|||
|
|
|
|||
|
|
@@ -965,6 +911,11 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
return new ConsumeQueueIterator(sbr);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
+ public ReferredIterator<CqUnit> iterateFrom(long startIndex, int count) {
|
|||
|
|
+ return iterateFrom(startIndex);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
@Override
|
|||
|
|
public CqUnit get(long offset) {
|
|||
|
|
ReferredIterator<CqUnit> it = iterateFrom(offset);
|
|||
|
|
@@ -974,6 +925,20 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
|
|||
|
|
return it.nextAndRelease();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
+ public Pair<CqUnit, Long> getCqUnitAndStoreTime(long index) {
|
|||
|
|
+ CqUnit cqUnit = get(index);
|
|||
|
|
+ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit);
|
|||
|
|
+ return new Pair<>(cqUnit, messageStoreTime);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public Pair<CqUnit, Long> getEarliestUnitAndStoreTime() {
|
|||
|
|
+ CqUnit cqUnit = getEarliestUnit();
|
|||
|
|
+ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit);
|
|||
|
|
+ return new Pair<>(cqUnit, messageStoreTime);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
@Override
|
|||
|
|
public CqUnit getEarliestUnit() {
|
|||
|
|
/**
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
index 02ea47f13..99a54e2d7 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
|
|||
|
|
@@ -105,32 +105,35 @@ import org.apache.rocketmq.store.logfile.MappedFile;
|
|||
|
|
import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager;
|
|||
|
|
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
|
|||
|
|
import org.apache.rocketmq.store.queue.ConsumeQueueStore;
|
|||
|
|
+import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;
|
|||
|
|
import org.apache.rocketmq.store.queue.CqUnit;
|
|||
|
|
import org.apache.rocketmq.store.queue.ReferredIterator;
|
|||
|
|
import org.apache.rocketmq.store.stats.BrokerStatsManager;
|
|||
|
|
import org.apache.rocketmq.store.timer.TimerMessageStore;
|
|||
|
|
import org.apache.rocketmq.store.util.PerfCounter;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
|
|||
|
|
public class DefaultMessageStore implements MessageStore {
|
|||
|
|
- private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
|
|||
|
|
+ protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
|
|||
|
|
+ protected static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);
|
|||
|
|
|
|||
|
|
public final PerfCounter.Ticks perfs = new PerfCounter.Ticks(LOGGER);
|
|||
|
|
|
|||
|
|
private final MessageStoreConfig messageStoreConfig;
|
|||
|
|
// CommitLog
|
|||
|
|
- private final CommitLog commitLog;
|
|||
|
|
+ protected final CommitLog commitLog;
|
|||
|
|
|
|||
|
|
- private final ConsumeQueueStore consumeQueueStore;
|
|||
|
|
+ protected final ConsumeQueueStoreInterface consumeQueueStore;
|
|||
|
|
|
|||
|
|
private final FlushConsumeQueueService flushConsumeQueueService;
|
|||
|
|
|
|||
|
|
- private final CleanCommitLogService cleanCommitLogService;
|
|||
|
|
+ protected final CleanCommitLogService cleanCommitLogService;
|
|||
|
|
|
|||
|
|
private final CleanConsumeQueueService cleanConsumeQueueService;
|
|||
|
|
|
|||
|
|
private final CorrectLogicOffsetService correctLogicOffsetService;
|
|||
|
|
|
|||
|
|
- private final IndexService indexService;
|
|||
|
|
+ protected final IndexService indexService;
|
|||
|
|
|
|||
|
|
private final AllocateMappedFileService allocateMappedFileService;
|
|||
|
|
|
|||
|
|
@@ -147,7 +150,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
private final TransientStorePool transientStorePool;
|
|||
|
|
|
|||
|
|
- private final RunningFlags runningFlags = new RunningFlags();
|
|||
|
|
+ protected final RunningFlags runningFlags = new RunningFlags();
|
|||
|
|
private final SystemClock systemClock = new SystemClock();
|
|||
|
|
|
|||
|
|
private final ScheduledExecutorService scheduledExecutorService;
|
|||
|
|
@@ -156,6 +159,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
private final BrokerConfig brokerConfig;
|
|||
|
|
|
|||
|
|
private volatile boolean shutdown = true;
|
|||
|
|
+ protected boolean notifyMessageArriveInBatch = false;
|
|||
|
|
|
|||
|
|
private StoreCheckpoint storeCheckpoint;
|
|||
|
|
private TimerMessageStore timerMessageStore;
|
|||
|
|
@@ -182,7 +186,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
private volatile long brokerInitMaxOffset = -1L;
|
|||
|
|
|
|||
|
|
- protected List<PutMessageHook> putMessageHookList = new ArrayList<>();
|
|||
|
|
+ private List<PutMessageHook> putMessageHookList = new ArrayList<>();
|
|||
|
|
|
|||
|
|
private SendMessageBackHook sendMessageBackHook;
|
|||
|
|
|
|||
|
|
@@ -222,12 +226,12 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
this.commitLog = new CommitLog(this);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- this.consumeQueueStore = new ConsumeQueueStore(this, this.messageStoreConfig);
|
|||
|
|
+ this.consumeQueueStore = createConsumeQueueStore();
|
|||
|
|
|
|||
|
|
- this.flushConsumeQueueService = new FlushConsumeQueueService();
|
|||
|
|
+ this.flushConsumeQueueService = createFlushConsumeQueueService();
|
|||
|
|
this.cleanCommitLogService = new CleanCommitLogService();
|
|||
|
|
- this.cleanConsumeQueueService = new CleanConsumeQueueService();
|
|||
|
|
- this.correctLogicOffsetService = new CorrectLogicOffsetService();
|
|||
|
|
+ this.cleanConsumeQueueService = createCleanConsumeQueueService();
|
|||
|
|
+ this.correctLogicOffsetService = createCorrectLogicOffsetService();
|
|||
|
|
this.storeStatsService = new StoreStatsService(getBrokerIdentity());
|
|||
|
|
this.indexService = new IndexService(this);
|
|||
|
|
|
|||
|
|
@@ -273,6 +277,22 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
parseDelayLevel();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ public ConsumeQueueStoreInterface createConsumeQueueStore() {
|
|||
|
|
+ return new ConsumeQueueStore(this);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CleanConsumeQueueService createCleanConsumeQueueService() {
|
|||
|
|
+ return new CleanConsumeQueueService();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public FlushConsumeQueueService createFlushConsumeQueueService() {
|
|||
|
|
+ return new FlushConsumeQueueService();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public CorrectLogicOffsetService createCorrectLogicOffsetService() {
|
|||
|
|
+ return new CorrectLogicOffsetService();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
public boolean parseDelayLevel() {
|
|||
|
|
HashMap<String, Long> timeUnitTable = new HashMap<>();
|
|||
|
|
timeUnitTable.put("s", 1000L);
|
|||
|
|
@@ -305,7 +325,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public void truncateDirtyLogicFiles(long phyOffset) {
|
|||
|
|
+ public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException {
|
|||
|
|
this.consumeQueueStore.truncateDirty(phyOffset);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -393,6 +413,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
this.flushConsumeQueueService.start();
|
|||
|
|
this.commitLog.start();
|
|||
|
|
+ this.consumeQueueStore.start();
|
|||
|
|
this.storeStatsService.start();
|
|||
|
|
|
|||
|
|
if (this.haService != null) {
|
|||
|
|
@@ -481,6 +502,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
this.storeStatsService.shutdown();
|
|||
|
|
this.commitLog.shutdown();
|
|||
|
|
this.reputMessageService.shutdown();
|
|||
|
|
+ this.consumeQueueStore.shutdown();
|
|||
|
|
// dispatch-related services must be shut down after reputMessageService
|
|||
|
|
this.indexService.shutdown();
|
|||
|
|
if (this.compactionService != null) {
|
|||
|
|
@@ -515,7 +537,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
public void destroy() {
|
|||
|
|
- this.destroyLogics();
|
|||
|
|
+ this.consumeQueueStore.destroy();
|
|||
|
|
this.commitLog.destroy();
|
|||
|
|
this.indexService.destroy();
|
|||
|
|
this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir()));
|
|||
|
|
@@ -541,11 +563,6 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
return commitLogSize + consumeQueueSize + indexFileSize;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- @Override
|
|||
|
|
- public void destroyLogics() {
|
|||
|
|
- this.consumeQueueStore.destroy();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
@Override
|
|||
|
|
public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner msg) {
|
|||
|
|
|
|||
|
|
@@ -687,7 +704,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
return commitLog;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public void truncateDirtyFiles(long offsetToTruncate) {
|
|||
|
|
+ public void truncateDirtyFiles(long offsetToTruncate) throws RocksDBException {
|
|||
|
|
|
|||
|
|
LOGGER.info("truncate dirty files to {}", offsetToTruncate);
|
|||
|
|
|
|||
|
|
@@ -700,12 +717,12 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
long oldReputFromOffset = this.reputMessageService.getReputFromOffset();
|
|||
|
|
|
|||
|
|
- // truncate commitLog
|
|||
|
|
- this.commitLog.truncateDirtyFiles(offsetToTruncate);
|
|||
|
|
-
|
|||
|
|
// truncate consume queue
|
|||
|
|
this.truncateDirtyLogicFiles(offsetToTruncate);
|
|||
|
|
|
|||
|
|
+ // truncate commitLog
|
|||
|
|
+ this.commitLog.truncateDirtyFiles(offsetToTruncate);
|
|||
|
|
+
|
|||
|
|
this.recoverTopicQueueTable();
|
|||
|
|
|
|||
|
|
if (!messageStoreConfig.isEnableBuildConsumeQueueConcurrently()) {
|
|||
|
|
@@ -723,7 +740,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public boolean truncateFiles(long offsetToTruncate) {
|
|||
|
|
+ public boolean truncateFiles(long offsetToTruncate) throws RocksDBException {
|
|||
|
|
if (offsetToTruncate >= this.getMaxPhyOffset()) {
|
|||
|
|
LOGGER.info("no need to truncate files, truncate offset is {}, max physical offset is {}", offsetToTruncate, this.getMaxPhyOffset());
|
|||
|
|
return true;
|
|||
|
|
@@ -825,17 +842,19 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
while (getResult.getBufferTotalSize() <= 0
|
|||
|
|
&& nextBeginOffset < maxOffset
|
|||
|
|
&& cqFileNum++ < this.messageStoreConfig.getTravelCqFileNumWhenGetMessage()) {
|
|||
|
|
- ReferredIterator<CqUnit> bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset);
|
|||
|
|
-
|
|||
|
|
- if (bufferConsumeQueue == null) {
|
|||
|
|
- status = GetMessageStatus.OFFSET_FOUND_NULL;
|
|||
|
|
- nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset));
|
|||
|
|
- LOGGER.warn("consumer request topic: " + topic + "offset: " + offset + " minOffset: " + minOffset + " maxOffset: "
|
|||
|
|
- + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset);
|
|||
|
|
- break;
|
|||
|
|
- }
|
|||
|
|
+ ReferredIterator<CqUnit> bufferConsumeQueue = null;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
+ bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset, maxMsgNums);
|
|||
|
|
+
|
|||
|
|
+ if (bufferConsumeQueue == null) {
|
|||
|
|
+ status = GetMessageStatus.OFFSET_FOUND_NULL;
|
|||
|
|
+ nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset));
|
|||
|
|
+ LOGGER.warn("consumer request topic: " + topic + ", offset: " + offset + ", minOffset: " + minOffset + ", maxOffset: "
|
|||
|
|
+ + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset);
|
|||
|
|
+ break;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
long nextPhyFileStartOffset = Long.MIN_VALUE;
|
|||
|
|
while (bufferConsumeQueue.hasNext()
|
|||
|
|
&& nextBeginOffset < maxOffset) {
|
|||
|
|
@@ -905,8 +924,13 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
status = GetMessageStatus.FOUND;
|
|||
|
|
nextPhyFileStartOffset = Long.MIN_VALUE;
|
|||
|
|
}
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("getMessage Failed. cid: {}, topic: {}, queueId: {}, offset: {}, minOffset: {}, maxOffset: {}, {}",
|
|||
|
|
+ group, topic, queueId, offset, minOffset, maxOffset, e.getMessage());
|
|||
|
|
} finally {
|
|||
|
|
- bufferConsumeQueue.release();
|
|||
|
|
+ if (bufferConsumeQueue != null) {
|
|||
|
|
+ bufferConsumeQueue.release();
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -975,12 +999,12 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
public long getMinOffsetInQueue(String topic, int queueId) {
|
|||
|
|
- ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId);
|
|||
|
|
- if (logic != null) {
|
|||
|
|
- return logic.getMinOffsetInQueue();
|
|||
|
|
+ try {
|
|||
|
|
+ return this.consumeQueueStore.getMinOffsetInQueue(topic, queueId);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("getMinOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e);
|
|||
|
|
+ return -1;
|
|||
|
|
}
|
|||
|
|
-
|
|||
|
|
- return -1;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
@@ -997,38 +1021,27 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) {
|
|||
|
|
ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId);
|
|||
|
|
if (consumeQueue != null) {
|
|||
|
|
-
|
|||
|
|
- ReferredIterator<CqUnit> bufferConsumeQueue = consumeQueue.iterateFrom(consumeQueueOffset);
|
|||
|
|
- if (bufferConsumeQueue != null) {
|
|||
|
|
- try {
|
|||
|
|
- if (bufferConsumeQueue.hasNext()) {
|
|||
|
|
- long offsetPy = bufferConsumeQueue.next().getPos();
|
|||
|
|
- return offsetPy;
|
|||
|
|
- }
|
|||
|
|
- } finally {
|
|||
|
|
- bufferConsumeQueue.release();
|
|||
|
|
- }
|
|||
|
|
+ CqUnit cqUnit = consumeQueue.get(consumeQueueOffset);
|
|||
|
|
+ if (cqUnit != null) {
|
|||
|
|
+ return cqUnit.getPos();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
-
|
|||
|
|
return 0;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) {
|
|||
|
|
- return getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER);
|
|||
|
|
+ return this.getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) {
|
|||
|
|
- ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId);
|
|||
|
|
- if (logic != null) {
|
|||
|
|
- long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType);
|
|||
|
|
- // Make sure the result offset is in valid range.
|
|||
|
|
- resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue());
|
|||
|
|
- resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue());
|
|||
|
|
- return resultOffset;
|
|||
|
|
+ try {
|
|||
|
|
+ return this.consumeQueueStore.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("getOffsetInQueueByTime Failed. topic: {}, queueId: {}, timestamp: {} boundaryType: {}, {}",
|
|||
|
|
+ topic, queueId, timestamp, boundaryType, e.getMessage());
|
|||
|
|
}
|
|||
|
|
-
|
|||
|
|
return 0;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -1088,6 +1101,10 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
return StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir());
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ public MessageArrivingListener getMessageArrivingListener() {
|
|||
|
|
+ return messageArrivingListener;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
@Override
|
|||
|
|
public HashMap<String, String> getRuntimeInfo() {
|
|||
|
|
HashMap<String, String> result = this.storeStatsService.getRuntimeInfo();
|
|||
|
|
@@ -1121,7 +1138,6 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
return this.commitLog.getMaxOffset();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
-
|
|||
|
|
@Override
|
|||
|
|
public long getMinPhyOffset() {
|
|||
|
|
return this.commitLog.getMinOffset();
|
|||
|
|
@@ -1141,7 +1157,10 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
public long getEarliestMessageTime(String topic, int queueId) {
|
|||
|
|
ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId);
|
|||
|
|
if (logicQueue != null) {
|
|||
|
|
- return getStoreTime(logicQueue.getEarliestUnit());
|
|||
|
|
+ Pair<CqUnit, Long> pair = logicQueue.getEarliestUnitAndStoreTime();
|
|||
|
|
+ if (pair != null && pair.getObject2() != null) {
|
|||
|
|
+ return pair.getObject2();
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return -1;
|
|||
|
|
@@ -1152,19 +1171,6 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
return CompletableFuture.completedFuture(getEarliestMessageTime(topic, queueId));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- protected long getStoreTime(CqUnit result) {
|
|||
|
|
- if (result != null) {
|
|||
|
|
- try {
|
|||
|
|
- final long phyOffset = result.getPos();
|
|||
|
|
- final int size = result.getSize();
|
|||
|
|
- long storeTime = this.getCommitLog().pickupStoreTimestamp(phyOffset, size);
|
|||
|
|
- return storeTime;
|
|||
|
|
- } catch (Exception e) {
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
- return -1;
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
@Override
|
|||
|
|
public long getEarliestMessageTime() {
|
|||
|
|
long minPhyOffset = this.getMinPhyOffset();
|
|||
|
|
@@ -1179,13 +1185,16 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) {
|
|||
|
|
ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId);
|
|||
|
|
if (logicQueue != null) {
|
|||
|
|
- return getStoreTime(logicQueue.get(consumeQueueOffset));
|
|||
|
|
+ Pair<CqUnit, Long> pair = logicQueue.getCqUnitAndStoreTime(consumeQueueOffset);
|
|||
|
|
+ if (pair != null && pair.getObject2() != null) {
|
|||
|
|
+ return pair.getObject2();
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
-
|
|||
|
|
return -1;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- @Override public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId,
|
|||
|
|
+ @Override
|
|||
|
|
+ public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId,
|
|||
|
|
long consumeQueueOffset) {
|
|||
|
|
return CompletableFuture.completedFuture(getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset));
|
|||
|
|
}
|
|||
|
|
@@ -1354,6 +1363,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
* If offset table is cleaned, and old messages are dispatching after the old consume queue is cleaned,
|
|||
|
|
* consume queue will be created with old offset, then later message with new offset table can not be
|
|||
|
|
* dispatched to consume queue.
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
*/
|
|||
|
|
@Override
|
|||
|
|
public int deleteTopics(final Set<String> deleteTopics) {
|
|||
|
|
@@ -1363,17 +1373,19 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
int deleteCount = 0;
|
|||
|
|
for (String topic : deleteTopics) {
|
|||
|
|
- ConcurrentMap<Integer, ConsumeQueueInterface> queueTable =
|
|||
|
|
- this.consumeQueueStore.getConsumeQueueTable().get(topic);
|
|||
|
|
+ ConcurrentMap<Integer, ConsumeQueueInterface> queueTable = this.consumeQueueStore.findConsumeQueueMap(topic);
|
|||
|
|
|
|||
|
|
if (queueTable == null || queueTable.isEmpty()) {
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for (ConsumeQueueInterface cq : queueTable.values()) {
|
|||
|
|
- this.consumeQueueStore.destroy(cq);
|
|||
|
|
- LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}",
|
|||
|
|
- cq.getTopic(), cq.getQueueId());
|
|||
|
|
+ try {
|
|||
|
|
+ this.consumeQueueStore.destroy(cq);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ LOGGER.error("DeleteTopic: ConsumeQueue cleans error!, topic={}, queueId={}", cq.getTopic(), cq.getQueueId(), e);
|
|||
|
|
+ }
|
|||
|
|
+ LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}", cq.getTopic(), cq.getQueueId());
|
|||
|
|
this.consumeQueueStore.removeTopicQueueTable(cq.getTopic(), cq.getQueueId());
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -1852,14 +1864,18 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
return file.exists();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private void recover(final boolean lastExitOK) {
|
|||
|
|
- boolean recoverConcurrently = this.brokerConfig.isRecoverConcurrently();
|
|||
|
|
+ private boolean isRecoverConcurrently() {
|
|||
|
|
+ return this.brokerConfig.isRecoverConcurrently() && !this.messageStoreConfig.isEnableRocksDBStore();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void recover(final boolean lastExitOK) throws RocksDBException {
|
|||
|
|
+ boolean recoverConcurrently = this.isRecoverConcurrently();
|
|||
|
|
LOGGER.info("message store recover mode: {}", recoverConcurrently ? "concurrent" : "normal");
|
|||
|
|
|
|||
|
|
// recover consume queue
|
|||
|
|
long recoverConsumeQueueStart = System.currentTimeMillis();
|
|||
|
|
this.recoverConsumeQueue();
|
|||
|
|
- long maxPhyOffsetOfConsumeQueue = this.getMaxOffsetInConsumeQueue();
|
|||
|
|
+ long maxPhyOffsetOfConsumeQueue = this.consumeQueueStore.getMaxPhyOffsetInConsumeQueue();
|
|||
|
|
long recoverConsumeQueueEnd = System.currentTimeMillis();
|
|||
|
|
|
|||
|
|
// recover commitlog
|
|||
|
|
@@ -1894,23 +1910,25 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
return messageStoreConfig;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
+ public void finishCommitLogDispatch() {
|
|||
|
|
+ // ignore
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
@Override
|
|||
|
|
public TransientStorePool getTransientStorePool() {
|
|||
|
|
return transientStorePool;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private void recoverConsumeQueue() {
|
|||
|
|
- if (!this.brokerConfig.isRecoverConcurrently()) {
|
|||
|
|
+ if (!this.isRecoverConcurrently()) {
|
|||
|
|
this.consumeQueueStore.recover();
|
|||
|
|
} else {
|
|||
|
|
this.consumeQueueStore.recoverConcurrently();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private long getMaxOffsetInConsumeQueue() {
|
|||
|
|
- return this.consumeQueueStore.getMaxOffsetInConsumeQueue();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
+ @Override
|
|||
|
|
public void recoverTopicQueueTable() {
|
|||
|
|
long minPhyOffset = this.commitLog.getMinOffset();
|
|||
|
|
this.consumeQueueStore.recoverOffsetTable(minPhyOffset);
|
|||
|
|
@@ -1949,13 +1967,17 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
return runningFlags;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public void doDispatch(DispatchRequest req) {
|
|||
|
|
+ public void doDispatch(DispatchRequest req) throws RocksDBException {
|
|||
|
|
for (CommitLogDispatcher dispatcher : this.dispatcherList) {
|
|||
|
|
dispatcher.dispatch(req);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public void putMessagePositionInfo(DispatchRequest dispatchRequest) {
|
|||
|
|
+ /**
|
|||
|
|
+ * @param dispatchRequest
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ protected void putMessagePositionInfo(DispatchRequest dispatchRequest) throws RocksDBException {
|
|||
|
|
this.consumeQueueStore.putMessagePositionInfoWrapper(dispatchRequest);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -2054,7 +2076,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public ConsumeQueueStore getQueueStore() {
|
|||
|
|
+ public ConsumeQueueStoreInterface getQueueStore() {
|
|||
|
|
return consumeQueueStore;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -2065,7 +2087,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile,
|
|||
|
|
- boolean isRecover, boolean isFileEnd) {
|
|||
|
|
+ boolean isRecover, boolean isFileEnd) throws RocksDBException {
|
|||
|
|
if (doDispatch && !isFileEnd) {
|
|||
|
|
this.doDispatch(dispatchRequest);
|
|||
|
|
}
|
|||
|
|
@@ -2082,7 +2104,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public void assignOffset(MessageExtBrokerInner msg) {
|
|||
|
|
+ public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException {
|
|||
|
|
final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
|
|||
|
|
|
|||
|
|
if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) {
|
|||
|
|
@@ -2127,12 +2149,12 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher {
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public void dispatch(DispatchRequest request) {
|
|||
|
|
+ public void dispatch(DispatchRequest request) throws RocksDBException {
|
|||
|
|
final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag());
|
|||
|
|
switch (tranType) {
|
|||
|
|
case MessageSysFlag.TRANSACTION_NOT_TYPE:
|
|||
|
|
case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
|
|||
|
|
- DefaultMessageStore.this.putMessagePositionInfo(request);
|
|||
|
|
+ putMessagePositionInfo(request);
|
|||
|
|
break;
|
|||
|
|
case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
|
|||
|
|
case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
|
|||
|
|
@@ -2278,7 +2300,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
return DefaultMessageStore.this.brokerConfig.getIdentifier() + CleanCommitLogService.class.getSimpleName();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private boolean isTimeToDelete() {
|
|||
|
|
+ protected boolean isTimeToDelete() {
|
|||
|
|
String when = DefaultMessageStore.this.getMessageStoreConfig().getDeleteWhen();
|
|||
|
|
if (UtilAll.isItTimeToDo(when)) {
|
|||
|
|
DefaultMessageStore.LOGGER.info("it's time to reclaim disk space, " + when);
|
|||
|
|
@@ -2436,7 +2458,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
class CleanConsumeQueueService {
|
|||
|
|
- private long lastPhysicalMinOffset = 0;
|
|||
|
|
+ protected long lastPhysicalMinOffset = 0;
|
|||
|
|
|
|||
|
|
public void run() {
|
|||
|
|
try {
|
|||
|
|
@@ -2446,7 +2468,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private void deleteExpiredFiles() {
|
|||
|
|
+ protected void deleteExpiredFiles() {
|
|||
|
|
int deleteLogicsFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval();
|
|||
|
|
|
|||
|
|
long minOffset = DefaultMessageStore.this.commitLog.getMinOffset();
|
|||
|
|
@@ -2551,7 +2573,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
if (cqUnit.getPos() >= minPhyOffset) {
|
|||
|
|
|
|||
|
|
- // Normal case, do not need correct.
|
|||
|
|
+ // Normal case, do not need to correct.
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
@@ -2741,6 +2763,18 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
+ public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) {
|
|||
|
|
+ if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable()
|
|||
|
|
+ && DefaultMessageStore.this.messageArrivingListener != null) {
|
|||
|
|
+ DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
|
|||
|
|
+ dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1,
|
|||
|
|
+ dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(),
|
|||
|
|
+ dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap());
|
|||
|
|
+ DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
class ReputMessageService extends ServiceThread {
|
|||
|
|
|
|||
|
|
protected volatile long reputFromOffset = 0;
|
|||
|
|
@@ -2810,13 +2844,8 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
if (size > 0) {
|
|||
|
|
DefaultMessageStore.this.doDispatch(dispatchRequest);
|
|||
|
|
|
|||
|
|
- if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable()
|
|||
|
|
- && DefaultMessageStore.this.messageArrivingListener != null) {
|
|||
|
|
- DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
|
|||
|
|
- dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1,
|
|||
|
|
- dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(),
|
|||
|
|
- dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap());
|
|||
|
|
- notifyMessageArrive4MultiQueue(dispatchRequest);
|
|||
|
|
+ if (!notifyMessageArriveInBatch) {
|
|||
|
|
+ notifyMessageArriveIfNecessary(dispatchRequest);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
this.reputFromOffset += size;
|
|||
|
|
@@ -2850,9 +2879,14 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.info("dispatch message to cq exception. reputFromOffset: {}", this.reputFromOffset, e);
|
|||
|
|
+ return;
|
|||
|
|
} finally {
|
|||
|
|
result.release();
|
|||
|
|
}
|
|||
|
|
+
|
|||
|
|
+ finishCommitLogDispatch();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -2989,7 +3023,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
// dispatchRequestsList:[
|
|||
|
|
// {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]},
|
|||
|
|
// {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}]
|
|||
|
|
- private void dispatch() {
|
|||
|
|
+ private void dispatch() throws Exception {
|
|||
|
|
dispatchRequestsList.clear();
|
|||
|
|
dispatchRequestOrderlyQueue.get(dispatchRequestsList);
|
|||
|
|
if (!dispatchRequestsList.isEmpty()) {
|
|||
|
|
@@ -2997,21 +3031,15 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
for (DispatchRequest dispatchRequest : dispatchRequests) {
|
|||
|
|
DefaultMessageStore.this.doDispatch(dispatchRequest);
|
|||
|
|
// wake up long-polling
|
|||
|
|
- if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable()
|
|||
|
|
- && DefaultMessageStore.this.messageArrivingListener != null) {
|
|||
|
|
- DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
|
|||
|
|
- dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1,
|
|||
|
|
- dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(),
|
|||
|
|
- dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap());
|
|||
|
|
- DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest);
|
|||
|
|
- }
|
|||
|
|
+ DefaultMessageStore.this.notifyMessageArriveIfNecessary(dispatchRequest);
|
|||
|
|
+
|
|||
|
|
if (!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() &&
|
|||
|
|
- DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
|
|||
|
|
+ DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
|
|||
|
|
DefaultMessageStore.this.storeStatsService
|
|||
|
|
- .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1);
|
|||
|
|
+ .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1);
|
|||
|
|
DefaultMessageStore.this.storeStatsService
|
|||
|
|
- .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())
|
|||
|
|
- .add(dispatchRequest.getMsgSize());
|
|||
|
|
+ .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())
|
|||
|
|
+ .add(dispatchRequest.getMsgSize());
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
@@ -3079,7 +3107,7 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
public void doReput() {
|
|||
|
|
if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) {
|
|||
|
|
LOGGER.warn("The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.",
|
|||
|
|
- this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset());
|
|||
|
|
+ this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset());
|
|||
|
|
this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset();
|
|||
|
|
}
|
|||
|
|
for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) {
|
|||
|
|
@@ -3138,6 +3166,9 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
result.release();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
+
|
|||
|
|
+ // only for rocksdb mode
|
|||
|
|
+ finishCommitLogDispatch();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
@@ -3180,8 +3211,8 @@ public class DefaultMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
if (this.isCommitLogAvailable()) {
|
|||
|
|
LOGGER.warn("shutdown concurrentReputMessageService, but CommitLog have not finish to be dispatched, CommitLog max" +
|
|||
|
|
- " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(),
|
|||
|
|
- this.reputFromOffset);
|
|||
|
|
+ " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(),
|
|||
|
|
+ this.reputFromOffset);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
this.mainBatchDispatchRequestService.shutdown();
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
|
|||
|
|
index 989cbbe31..814c6d1bf 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
|
|||
|
|
@@ -16,10 +16,6 @@
|
|||
|
|
*/
|
|||
|
|
package org.apache.rocketmq.store;
|
|||
|
|
|
|||
|
|
-import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
-import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
-import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
-import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
import java.nio.ByteBuffer;
|
|||
|
|
import java.util.HashMap;
|
|||
|
|
import java.util.LinkedList;
|
|||
|
|
@@ -40,10 +36,15 @@ import org.apache.rocketmq.store.hook.PutMessageHook;
|
|||
|
|
import org.apache.rocketmq.store.hook.SendMessageBackHook;
|
|||
|
|
import org.apache.rocketmq.store.logfile.MappedFile;
|
|||
|
|
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
|
|||
|
|
-import org.apache.rocketmq.store.queue.ConsumeQueueStore;
|
|||
|
|
+import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;
|
|||
|
|
import org.apache.rocketmq.store.stats.BrokerStatsManager;
|
|||
|
|
import org.apache.rocketmq.store.timer.TimerMessageStore;
|
|||
|
|
import org.apache.rocketmq.store.util.PerfCounter;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
+import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* This class defines contracting interfaces to implement, allowing third-party vendor to use customized message store.
|
|||
|
|
@@ -545,7 +546,7 @@ public interface MessageStore {
|
|||
|
|
void setConfirmOffset(long phyOffset);
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
- * Check if the operation system page cache is busy or not.
|
|||
|
|
+ * Check if the operating system page cache is busy or not.
|
|||
|
|
*
|
|||
|
|
* @return true if the OS page cache is busy; false otherwise.
|
|||
|
|
*/
|
|||
|
|
@@ -620,9 +621,18 @@ public interface MessageStore {
|
|||
|
|
* @param commitLogFile commit log file
|
|||
|
|
* @param isRecover is from recover process
|
|||
|
|
* @param isFileEnd if the dispatch request represents 'file end'
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
*/
|
|||
|
|
void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile,
|
|||
|
|
- boolean isRecover, boolean isFileEnd);
|
|||
|
|
+ boolean isRecover, boolean isFileEnd) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Only used in rocksdb mode, because we build consumeQueue in batch(default 16 dispatchRequests)
|
|||
|
|
+ * It will be triggered in two cases:
|
|||
|
|
+ * @see org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput
|
|||
|
|
+ * @see CommitLog#recoverAbnormally
|
|||
|
|
+ */
|
|||
|
|
+ void finishCommitLogDispatch();
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Get the message store config
|
|||
|
|
@@ -691,13 +701,9 @@ public interface MessageStore {
|
|||
|
|
* Truncate dirty logic files
|
|||
|
|
*
|
|||
|
|
* @param phyOffset physical offset
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
*/
|
|||
|
|
- void truncateDirtyLogicFiles(long phyOffset);
|
|||
|
|
-
|
|||
|
|
- /**
|
|||
|
|
- * Destroy logics files
|
|||
|
|
- */
|
|||
|
|
- void destroyLogics();
|
|||
|
|
+ void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Unlock mappedFile
|
|||
|
|
@@ -718,7 +724,7 @@ public interface MessageStore {
|
|||
|
|
*
|
|||
|
|
* @return the queue store
|
|||
|
|
*/
|
|||
|
|
- ConsumeQueueStore getQueueStore();
|
|||
|
|
+ ConsumeQueueStoreInterface getQueueStore();
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* If 'sync disk flush' is configured in this message store
|
|||
|
|
@@ -739,8 +745,9 @@ public interface MessageStore {
|
|||
|
|
* yourself.
|
|||
|
|
*
|
|||
|
|
* @param msg message
|
|||
|
|
+ * @throws RocksDBException
|
|||
|
|
*/
|
|||
|
|
- void assignOffset(MessageExtBrokerInner msg);
|
|||
|
|
+ void assignOffset(MessageExtBrokerInner msg) throws RocksDBException;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Increase queue offset in memory table. If there is a race condition, you need to lock/unlock this method
|
|||
|
|
@@ -835,14 +842,15 @@ public interface MessageStore {
|
|||
|
|
*
|
|||
|
|
* @param offsetToTruncate offset to truncate
|
|||
|
|
* @return true if truncate succeed, false otherwise
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
*/
|
|||
|
|
- boolean truncateFiles(long offsetToTruncate);
|
|||
|
|
+ boolean truncateFiles(long offsetToTruncate) throws RocksDBException;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
- * Check if the offset is align with one message.
|
|||
|
|
+ * Check if the offset is aligned with one message.
|
|||
|
|
*
|
|||
|
|
* @param offset offset to check
|
|||
|
|
- * @return true if align, false otherwise
|
|||
|
|
+ * @return true if aligned, false otherwise
|
|||
|
|
*/
|
|||
|
|
boolean isOffsetAligned(long offset);
|
|||
|
|
|
|||
|
|
@@ -971,4 +979,14 @@ public interface MessageStore {
|
|||
|
|
* @param attributesBuilderSupplier metrics attributes builder
|
|||
|
|
*/
|
|||
|
|
void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Recover topic queue table
|
|||
|
|
+ */
|
|||
|
|
+ void recoverTopicQueueTable();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * notify message arrive if necessary
|
|||
|
|
+ */
|
|||
|
|
+ void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest);
|
|||
|
|
}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..87ccb5474
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java
|
|||
|
|
@@ -0,0 +1,169 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store;
|
|||
|
|
+
|
|||
|
|
+import java.io.IOException;
|
|||
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+
|
|||
|
|
+import org.apache.rocketmq.common.BrokerConfig;
|
|||
|
|
+import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
+import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
+import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
+import org.apache.rocketmq.store.config.StorePathConfigHelper;
|
|||
|
|
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
|
|||
|
|
+import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;
|
|||
|
|
+import org.apache.rocketmq.store.queue.RocksDBConsumeQueue;
|
|||
|
|
+import org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore;
|
|||
|
|
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+
|
|||
|
|
+public class RocksDBMessageStore extends DefaultMessageStore {
|
|||
|
|
+
|
|||
|
|
+ public RocksDBMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
|
|||
|
|
+ final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap<String, TopicConfig> topicConfigTable) throws
|
|||
|
|
+ IOException {
|
|||
|
|
+ super(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, topicConfigTable);
|
|||
|
|
+ notifyMessageArriveInBatch = true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ConsumeQueueStoreInterface createConsumeQueueStore() {
|
|||
|
|
+ return new RocksDBConsumeQueueStore(this);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CleanConsumeQueueService createCleanConsumeQueueService() {
|
|||
|
|
+ return new RocksDBCleanConsumeQueueService();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public FlushConsumeQueueService createFlushConsumeQueueService() {
|
|||
|
|
+ return new RocksDBFlushConsumeQueueService();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CorrectLogicOffsetService createCorrectLogicOffsetService() {
|
|||
|
|
+ return new RocksDBCorrectLogicOffsetService();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Try to set topicQueueTable = new HashMap<>(), otherwise it will cause bug when broker role changes.
|
|||
|
|
+ * And unlike method in DefaultMessageStore, we don't need to really recover topic queue table advance,
|
|||
|
|
+ * because we can recover topic queue table from rocksdb when we need to use it.
|
|||
|
|
+ * @see RocksDBConsumeQueue#assignQueueOffset
|
|||
|
|
+ */
|
|||
|
|
+ @Override
|
|||
|
|
+ public void recoverTopicQueueTable() {
|
|||
|
|
+ this.consumeQueueStore.setTopicQueueTable(new ConcurrentHashMap<>());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void finishCommitLogDispatch() {
|
|||
|
|
+ try {
|
|||
|
|
+ putMessagePositionInfo(null);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.info("try to finish commitlog dispatch error.", e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) {
|
|||
|
|
+ return findConsumeQueue(topic, queueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ class RocksDBCleanConsumeQueueService extends CleanConsumeQueueService {
|
|||
|
|
+ private final double diskSpaceWarningLevelRatio =
|
|||
|
|
+ Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceWarningLevelRatio", "0.90"));
|
|||
|
|
+
|
|||
|
|
+ private final double diskSpaceCleanForciblyRatio =
|
|||
|
|
+ Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceCleanForciblyRatio", "0.85"));
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ protected void deleteExpiredFiles() {
|
|||
|
|
+
|
|||
|
|
+ long minOffset = RocksDBMessageStore.this.commitLog.getMinOffset();
|
|||
|
|
+ if (minOffset > this.lastPhysicalMinOffset) {
|
|||
|
|
+ this.lastPhysicalMinOffset = minOffset;
|
|||
|
|
+
|
|||
|
|
+ boolean spaceFull = isSpaceToDelete();
|
|||
|
|
+ boolean timeUp = cleanCommitLogService.isTimeToDelete();
|
|||
|
|
+ if (spaceFull || timeUp) {
|
|||
|
|
+ RocksDBMessageStore.this.consumeQueueStore.cleanExpired(minOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ RocksDBMessageStore.this.indexService.deleteExpiredFile(minOffset);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private boolean isSpaceToDelete() {
|
|||
|
|
+ double ratio = RocksDBMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0;
|
|||
|
|
+
|
|||
|
|
+ String storePathLogics = StorePathConfigHelper
|
|||
|
|
+ .getStorePathConsumeQueue(RocksDBMessageStore.this.getMessageStoreConfig().getStorePathRootDir());
|
|||
|
|
+ double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics);
|
|||
|
|
+ if (logicsRatio > diskSpaceWarningLevelRatio) {
|
|||
|
|
+ boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskFull();
|
|||
|
|
+ if (diskOk) {
|
|||
|
|
+ RocksDBMessageStore.LOGGER.error("logics disk maybe full soon " + logicsRatio + ", so mark disk full");
|
|||
|
|
+ }
|
|||
|
|
+ } else if (logicsRatio > diskSpaceCleanForciblyRatio) {
|
|||
|
|
+ } else {
|
|||
|
|
+ boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskOK();
|
|||
|
|
+ if (!diskOk) {
|
|||
|
|
+ RocksDBMessageStore.LOGGER.info("logics disk space OK " + logicsRatio + ", so mark disk ok");
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ if (logicsRatio < 0 || logicsRatio > ratio) {
|
|||
|
|
+ RocksDBMessageStore.LOGGER.info("logics disk maybe full soon, so reclaim space, " + logicsRatio);
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ class RocksDBFlushConsumeQueueService extends FlushConsumeQueueService {
|
|||
|
|
+ /**
|
|||
|
|
+ * There is no need to flush consume queue,
|
|||
|
|
+ * we put all consume queues in RocksDBConsumeQueueStore,
|
|||
|
|
+ * it depends on rocksdb to flush consume queue to disk(sorted string table),
|
|||
|
|
+ * we even don't flush WAL of consume store, since we think it can recover consume queue from commitlog.
|
|||
|
|
+ */
|
|||
|
|
+ @Override
|
|||
|
|
+ public void run() {
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ class RocksDBCorrectLogicOffsetService extends CorrectLogicOffsetService {
|
|||
|
|
+ /**
|
|||
|
|
+ * There is no need to correct min offset of consume queue, we already fix this problem.
|
|||
|
|
+ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset
|
|||
|
|
+ */
|
|||
|
|
+ public void run() {
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter) {
|
|||
|
|
+ // todo
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java
|
|||
|
|
index 2ae6879aa..91fcb155a 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java
|
|||
|
|
@@ -30,6 +30,8 @@ public class RunningFlags {
|
|||
|
|
|
|||
|
|
private static final int FENCED_BIT = 1 << 5;
|
|||
|
|
|
|||
|
|
+ private static final int LOGIC_DISK_FULL_BIT = 1 << 5;
|
|||
|
|
+
|
|||
|
|
private volatile int flagBits = 0;
|
|||
|
|
|
|||
|
|
public RunningFlags() {
|
|||
|
|
@@ -63,6 +65,10 @@ public class RunningFlags {
|
|||
|
|
return result;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ public void clearLogicsQueueError() {
|
|||
|
|
+ this.flagBits &= ~WRITE_LOGICS_QUEUE_ERROR_BIT;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
public boolean getAndMakeWriteable() {
|
|||
|
|
boolean result = this.isWriteable();
|
|||
|
|
if (!result) {
|
|||
|
|
@@ -72,7 +78,7 @@ public class RunningFlags {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public boolean isWriteable() {
|
|||
|
|
- if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT)) == 0) {
|
|||
|
|
+ if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT | LOGIC_DISK_FULL_BIT)) == 0) {
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -81,7 +87,7 @@ public class RunningFlags {
|
|||
|
|
|
|||
|
|
//for consume queue, just ignore the DISK_FULL_BIT
|
|||
|
|
public boolean isCQWriteable() {
|
|||
|
|
- if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT)) == 0) {
|
|||
|
|
+ if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT | LOGIC_DISK_FULL_BIT)) == 0) {
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -139,4 +145,16 @@ public class RunningFlags {
|
|||
|
|
this.flagBits &= ~DISK_FULL_BIT;
|
|||
|
|
return result;
|
|||
|
|
}
|
|||
|
|
+
|
|||
|
|
+ public boolean getAndMakeLogicDiskFull() {
|
|||
|
|
+ boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT);
|
|||
|
|
+ this.flagBits |= LOGIC_DISK_FULL_BIT;
|
|||
|
|
+ return result;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public boolean getAndMakeLogicDiskOK() {
|
|||
|
|
+ boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT);
|
|||
|
|
+ this.flagBits &= ~LOGIC_DISK_FULL_BIT;
|
|||
|
|
+ return result;
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
|
|||
|
|
index 9fa448043..028facbdc 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
|
|||
|
|
@@ -397,8 +397,10 @@ public class MessageStoreConfig {
|
|||
|
|
private int batchDispatchRequestThreadPoolNums = 16;
|
|||
|
|
|
|||
|
|
// rocksdb mode
|
|||
|
|
+ private long cleanRocksDBDirtyCQIntervalMin = 60;
|
|||
|
|
+ private long statRocksDBCQIntervalSec = 10;
|
|||
|
|
+ private long memTableFlushIntervalMs = 60 * 60 * 1000L;
|
|||
|
|
private boolean realTimePersistRocksDBConfig = true;
|
|||
|
|
- private long memTableFlushInterval = 60 * 60 * 1000L;
|
|||
|
|
private boolean enableRocksDBLog = false;
|
|||
|
|
|
|||
|
|
private int topicQueueLockNum = 32;
|
|||
|
|
@@ -499,6 +501,10 @@ public class MessageStoreConfig {
|
|||
|
|
this.mappedFileSizeCommitLog = mappedFileSizeCommitLog;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ public boolean isEnableRocksDBStore() {
|
|||
|
|
+ return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.storeType);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
public String getStoreType() {
|
|||
|
|
return storeType;
|
|||
|
|
}
|
|||
|
|
@@ -508,7 +514,6 @@ public class MessageStoreConfig {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public int getMappedFileSizeConsumeQueue() {
|
|||
|
|
-
|
|||
|
|
int factor = (int) Math.ceil(this.mappedFileSizeConsumeQueue / (ConsumeQueue.CQ_STORE_UNIT_SIZE * 1.0));
|
|||
|
|
return (int) (factor * ConsumeQueue.CQ_STORE_UNIT_SIZE);
|
|||
|
|
}
|
|||
|
|
@@ -1738,12 +1743,28 @@ public class MessageStoreConfig {
|
|||
|
|
this.realTimePersistRocksDBConfig = realTimePersistRocksDBConfig;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public long getMemTableFlushInterval() {
|
|||
|
|
- return memTableFlushInterval;
|
|||
|
|
+ public long getStatRocksDBCQIntervalSec() {
|
|||
|
|
+ return statRocksDBCQIntervalSec;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void setStatRocksDBCQIntervalSec(long statRocksDBCQIntervalSec) {
|
|||
|
|
+ this.statRocksDBCQIntervalSec = statRocksDBCQIntervalSec;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long getCleanRocksDBDirtyCQIntervalMin() {
|
|||
|
|
+ return cleanRocksDBDirtyCQIntervalMin;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void setCleanRocksDBDirtyCQIntervalMin(long cleanRocksDBDirtyCQIntervalMin) {
|
|||
|
|
+ this.cleanRocksDBDirtyCQIntervalMin = cleanRocksDBDirtyCQIntervalMin;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long getMemTableFlushIntervalMs() {
|
|||
|
|
+ return memTableFlushIntervalMs;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public void setMemTableFlushInterval(long memTableFlushInterval) {
|
|||
|
|
- this.memTableFlushInterval = memTableFlushInterval;
|
|||
|
|
+ public void setMemTableFlushIntervalMs(long memTableFlushIntervalMs) {
|
|||
|
|
+ this.memTableFlushIntervalMs = memTableFlushIntervalMs;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public boolean isEnableRocksDBLog() {
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
|
|||
|
|
index d5f6acdc0..70371d83b 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
|
|||
|
|
@@ -16,26 +16,13 @@
|
|||
|
|
*/
|
|||
|
|
package org.apache.rocketmq.store.dledger;
|
|||
|
|
|
|||
|
|
-import io.openmessaging.storage.dledger.AppendFuture;
|
|||
|
|
-import io.openmessaging.storage.dledger.BatchAppendFuture;
|
|||
|
|
-import io.openmessaging.storage.dledger.DLedgerConfig;
|
|||
|
|
-import io.openmessaging.storage.dledger.DLedgerServer;
|
|||
|
|
-import io.openmessaging.storage.dledger.entry.DLedgerEntry;
|
|||
|
|
-import io.openmessaging.storage.dledger.protocol.AppendEntryRequest;
|
|||
|
|
-import io.openmessaging.storage.dledger.protocol.AppendEntryResponse;
|
|||
|
|
-import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest;
|
|||
|
|
-import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode;
|
|||
|
|
-import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore;
|
|||
|
|
-import io.openmessaging.storage.dledger.store.file.MmapFile;
|
|||
|
|
-import io.openmessaging.storage.dledger.store.file.MmapFileList;
|
|||
|
|
-import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult;
|
|||
|
|
-import io.openmessaging.storage.dledger.utils.DLedgerUtils;
|
|||
|
|
import java.net.Inet6Address;
|
|||
|
|
import java.net.InetSocketAddress;
|
|||
|
|
import java.nio.ByteBuffer;
|
|||
|
|
import java.util.LinkedList;
|
|||
|
|
import java.util.List;
|
|||
|
|
import java.util.concurrent.CompletableFuture;
|
|||
|
|
+
|
|||
|
|
import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageDecoder;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageExtBatch;
|
|||
|
|
@@ -54,6 +41,22 @@ import org.apache.rocketmq.store.SelectMappedBufferResult;
|
|||
|
|
import org.apache.rocketmq.store.StoreStatsService;
|
|||
|
|
import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
import org.apache.rocketmq.store.logfile.MappedFile;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+
|
|||
|
|
+import io.openmessaging.storage.dledger.AppendFuture;
|
|||
|
|
+import io.openmessaging.storage.dledger.BatchAppendFuture;
|
|||
|
|
+import io.openmessaging.storage.dledger.DLedgerConfig;
|
|||
|
|
+import io.openmessaging.storage.dledger.DLedgerServer;
|
|||
|
|
+import io.openmessaging.storage.dledger.entry.DLedgerEntry;
|
|||
|
|
+import io.openmessaging.storage.dledger.protocol.AppendEntryRequest;
|
|||
|
|
+import io.openmessaging.storage.dledger.protocol.AppendEntryResponse;
|
|||
|
|
+import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest;
|
|||
|
|
+import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode;
|
|||
|
|
+import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore;
|
|||
|
|
+import io.openmessaging.storage.dledger.store.file.MmapFile;
|
|||
|
|
+import io.openmessaging.storage.dledger.store.file.MmapFileList;
|
|||
|
|
+import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult;
|
|||
|
|
+import io.openmessaging.storage.dledger.utils.DLedgerUtils;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Store all metadata downtime for recovery, data protection reliability
|
|||
|
|
@@ -269,7 +272,7 @@ public class DLedgerCommitLog extends CommitLog {
|
|||
|
|
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
-
|
|||
|
|
+
|
|||
|
|
@Override
|
|||
|
|
public boolean getData(final long offset, final int size, final ByteBuffer byteBuffer) {
|
|||
|
|
if (offset < dividedCommitlogOffset) {
|
|||
|
|
@@ -287,7 +290,7 @@ public class DLedgerCommitLog extends CommitLog {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private void recover(long maxPhyOffsetOfConsumeQueue) {
|
|||
|
|
+ private void recover(long maxPhyOffsetOfConsumeQueue) throws RocksDBException {
|
|||
|
|
dLedgerFileStore.load();
|
|||
|
|
if (dLedgerFileList.getMappedFiles().size() > 0) {
|
|||
|
|
dLedgerFileStore.recover();
|
|||
|
|
@@ -341,12 +344,12 @@ public class DLedgerCommitLog extends CommitLog {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public void recoverNormally(long maxPhyOffsetOfConsumeQueue) {
|
|||
|
|
+ public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException {
|
|||
|
|
recover(maxPhyOffsetOfConsumeQueue);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) {
|
|||
|
|
+ public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException {
|
|||
|
|
recover(maxPhyOffsetOfConsumeQueue);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -469,9 +472,6 @@ public class DLedgerCommitLog extends CommitLog {
|
|||
|
|
String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset);
|
|||
|
|
elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock;
|
|||
|
|
appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.getData().length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock);
|
|||
|
|
- } catch (Exception e) {
|
|||
|
|
- log.error("Put message error", e);
|
|||
|
|
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));
|
|||
|
|
} finally {
|
|||
|
|
beginTimeInDledgerLock = 0;
|
|||
|
|
putMessageLock.unlock();
|
|||
|
|
@@ -482,6 +482,9 @@ public class DLedgerCommitLog extends CommitLog {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
defaultMessageStore.increaseOffset(msg, getMessageNum(msg));
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ log.error("Put message error", e);
|
|||
|
|
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));
|
|||
|
|
} finally {
|
|||
|
|
topicQueueLock.unlock(topicQueueKey);
|
|||
|
|
}
|
|||
|
|
@@ -611,9 +614,6 @@ public class DLedgerCommitLog extends CommitLog {
|
|||
|
|
appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen,
|
|||
|
|
msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock);
|
|||
|
|
appendResult.setMsgNum(msgNum);
|
|||
|
|
- } catch (Exception e) {
|
|||
|
|
- log.error("Put message error", e);
|
|||
|
|
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));
|
|||
|
|
} finally {
|
|||
|
|
beginTimeInDledgerLock = 0;
|
|||
|
|
putMessageLock.unlock();
|
|||
|
|
@@ -626,7 +626,10 @@ public class DLedgerCommitLog extends CommitLog {
|
|||
|
|
|
|||
|
|
defaultMessageStore.increaseOffset(messageExtBatch, (short) batchNum);
|
|||
|
|
|
|||
|
|
- } finally {
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ log.error("Put message error", e);
|
|||
|
|
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));
|
|||
|
|
+ } finally {
|
|||
|
|
topicQueueLock.unlock(encodeResult.queueOffsetKey);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
|
|||
|
|
index 467da603d..aaea7d690 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
|
|||
|
|
@@ -25,6 +25,7 @@ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
|
|||
|
|
import org.apache.rocketmq.store.CommitLog;
|
|||
|
|
import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
|
|||
|
|
public interface HAService {
|
|||
|
|
|
|||
|
|
@@ -53,7 +54,7 @@ public interface HAService {
|
|||
|
|
*
|
|||
|
|
* @param masterEpoch the new masterEpoch
|
|||
|
|
*/
|
|||
|
|
- default boolean changeToMaster(int masterEpoch) {
|
|||
|
|
+ default boolean changeToMaster(int masterEpoch) throws RocksDBException {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java
|
|||
|
|
index 936db0c4c..176c25a96 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java
|
|||
|
|
@@ -432,7 +432,7 @@ public class AutoSwitchHAClient extends ServiceThread implements HAClient {
|
|||
|
|
/**
|
|||
|
|
* Compare the master and slave's epoch file, find consistent point, do truncate.
|
|||
|
|
*/
|
|||
|
|
- private boolean doTruncate(List<EpochEntry> masterEpochEntries, long masterEndOffset) throws IOException {
|
|||
|
|
+ private boolean doTruncate(List<EpochEntry> masterEpochEntries, long masterEndOffset) throws Exception {
|
|||
|
|
if (this.epochCache.getEntrySize() == 0) {
|
|||
|
|
// If epochMap is empty, means the broker is a new replicas
|
|||
|
|
LOGGER.info("Slave local epochCache is empty, skip truncate log");
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
index f20bc3e28..64dad9aef 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java
|
|||
|
|
@@ -51,6 +51,7 @@ import org.apache.rocketmq.store.ha.GroupTransferService;
|
|||
|
|
import org.apache.rocketmq.store.ha.HAClient;
|
|||
|
|
import org.apache.rocketmq.store.ha.HAConnection;
|
|||
|
|
import org.apache.rocketmq.store.ha.HAConnectionStateNotificationService;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* SwitchAble ha service, support switch role to master or slave.
|
|||
|
|
@@ -111,7 +112,7 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public boolean changeToMaster(int masterEpoch) {
|
|||
|
|
+ public boolean changeToMaster(int masterEpoch) throws RocksDBException {
|
|||
|
|
final int lastEpoch = this.epochCache.lastEpoch();
|
|||
|
|
if (masterEpoch < lastEpoch) {
|
|||
|
|
LOGGER.warn("newMasterEpoch {} < lastEpoch {}, fail to change to master", masterEpoch, lastEpoch);
|
|||
|
|
@@ -315,7 +316,7 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
final EpochEntry currentLeaderEpoch = this.epochCache.lastEntry();
|
|||
|
|
if (slaveMaxOffset >= currentLeaderEpoch.getStartOffset()) {
|
|||
|
|
LOGGER.info("The slave {} has caught up, slaveMaxOffset: {}, confirmOffset: {}, epoch: {}, leader epoch startOffset: {}.",
|
|||
|
|
- slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset());
|
|||
|
|
+ slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset());
|
|||
|
|
currentSyncStateSet.add(slaveBrokerId);
|
|||
|
|
markSynchronizingSyncStateSet(currentSyncStateSet);
|
|||
|
|
// Notify the upper layer that syncStateSet changed.
|
|||
|
|
@@ -491,7 +492,7 @@ public class AutoSwitchHAService extends DefaultHAService {
|
|||
|
|
/**
|
|||
|
|
* Try to truncate incomplete msg transferred from master.
|
|||
|
|
*/
|
|||
|
|
- public long truncateInvalidMsg() {
|
|||
|
|
+ public long truncateInvalidMsg() throws RocksDBException {
|
|||
|
|
long dispatchBehind = this.defaultMessageStore.dispatchBehindBytes();
|
|||
|
|
if (dispatchBehind <= 0) {
|
|||
|
|
LOGGER.info("Dispatch complete, skip truncate");
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
|
|||
|
|
index ab9fc6da7..2f2ce9812 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
|
|||
|
|
@@ -17,10 +17,6 @@
|
|||
|
|
|
|||
|
|
package org.apache.rocketmq.store.plugin;
|
|||
|
|
|
|||
|
|
-import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
-import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
-import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
-import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
import java.nio.ByteBuffer;
|
|||
|
|
import java.util.HashMap;
|
|||
|
|
import java.util.LinkedList;
|
|||
|
|
@@ -55,10 +51,16 @@ import org.apache.rocketmq.store.hook.PutMessageHook;
|
|||
|
|
import org.apache.rocketmq.store.hook.SendMessageBackHook;
|
|||
|
|
import org.apache.rocketmq.store.logfile.MappedFile;
|
|||
|
|
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
|
|||
|
|
-import org.apache.rocketmq.store.queue.ConsumeQueueStore;
|
|||
|
|
+import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;
|
|||
|
|
import org.apache.rocketmq.store.stats.BrokerStatsManager;
|
|||
|
|
import org.apache.rocketmq.store.timer.TimerMessageStore;
|
|||
|
|
import org.apache.rocketmq.store.util.PerfCounter;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+
|
|||
|
|
+import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
+import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
|
|||
|
|
public abstract class AbstractPluginMessageStore implements MessageStore {
|
|||
|
|
protected MessageStore next = null;
|
|||
|
|
@@ -457,7 +459,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public boolean truncateFiles(long offsetToTruncate) {
|
|||
|
|
+ public boolean truncateFiles(long offsetToTruncate) throws RocksDBException {
|
|||
|
|
return next.truncateFiles(offsetToTruncate);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -511,7 +513,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile,
|
|||
|
|
- boolean isRecover, boolean isFileEnd) {
|
|||
|
|
+ boolean isRecover, boolean isFileEnd) throws RocksDBException {
|
|||
|
|
next.onCommitLogDispatch(dispatchRequest, doDispatch, commitLogFile, isRecover, isFileEnd);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -551,15 +553,10 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public void truncateDirtyLogicFiles(long phyOffset) {
|
|||
|
|
+ public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException {
|
|||
|
|
next.truncateDirtyLogicFiles(phyOffset);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- @Override
|
|||
|
|
- public void destroyLogics() {
|
|||
|
|
- next.destroyLogics();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
@Override
|
|||
|
|
public void unlockMappedFile(MappedFile unlockMappedFile) {
|
|||
|
|
next.unlockMappedFile(unlockMappedFile);
|
|||
|
|
@@ -571,7 +568,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public ConsumeQueueStore getQueueStore() {
|
|||
|
|
+ public ConsumeQueueStoreInterface getQueueStore() {
|
|||
|
|
return next.getQueueStore();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -586,7 +583,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Override
|
|||
|
|
- public void assignOffset(MessageExtBrokerInner msg) {
|
|||
|
|
+ public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException {
|
|||
|
|
next.assignOffset(msg);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -649,4 +646,19 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
|
|||
|
|
public void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier) {
|
|||
|
|
next.initMetrics(meter, attributesBuilderSupplier);
|
|||
|
|
}
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void finishCommitLogDispatch() {
|
|||
|
|
+ next.finishCommitLogDispatch();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void recoverTopicQueueTable() {
|
|||
|
|
+ next.recoverTopicQueueTable();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) {
|
|||
|
|
+ next.notifyMessageArriveIfNecessary(dispatchRequest);
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..30054fa50
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java
|
|||
|
|
@@ -0,0 +1,105 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.queue;
|
|||
|
|
+
|
|||
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
+import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
+import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+
|
|||
|
|
+public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface {
|
|||
|
|
+ protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
+ protected final DefaultMessageStore messageStore;
|
|||
|
|
+ protected final MessageStoreConfig messageStoreConfig;
|
|||
|
|
+ protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator();
|
|||
|
|
+ protected final ConcurrentMap<String/* topic */, ConcurrentMap<Integer/* queueId */, ConsumeQueueInterface>> consumeQueueTable;
|
|||
|
|
+
|
|||
|
|
+ public AbstractConsumeQueueStore(DefaultMessageStore messageStore) {
|
|||
|
|
+ this.messageStore = messageStore;
|
|||
|
|
+ this.messageStoreConfig = messageStore.getMessageStoreConfig();
|
|||
|
|
+ this.consumeQueueTable = new ConcurrentHashMap<>(32);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) {
|
|||
|
|
+ consumeQueue.putMessagePositionInfoWrapper(request);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public Long getMaxOffset(String topic, int queueId) {
|
|||
|
|
+ return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void setTopicQueueTable(ConcurrentMap<String, Long> topicQueueTable) {
|
|||
|
|
+ this.queueOffsetOperator.setTopicQueueTable(topicQueueTable);
|
|||
|
|
+ this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ConcurrentMap getTopicQueueTable() {
|
|||
|
|
+ return this.queueOffsetOperator.getTopicQueueTable();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException {
|
|||
|
|
+ ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId());
|
|||
|
|
+ consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) {
|
|||
|
|
+ ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId());
|
|||
|
|
+ consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void removeTopicQueueTable(String topic, Integer queueId) {
|
|||
|
|
+ this.queueOffsetOperator.remove(topic, queueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> getConsumeQueueTable() {
|
|||
|
|
+ return this.consumeQueueTable;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ConcurrentMap<Integer, ConsumeQueueInterface> findConsumeQueueMap(String topic) {
|
|||
|
|
+ return this.consumeQueueTable.get(topic);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getStoreTime(CqUnit cqUnit) {
|
|||
|
|
+ if (cqUnit != null) {
|
|||
|
|
+ try {
|
|||
|
|
+ final long phyOffset = cqUnit.getPos();
|
|||
|
|
+ final int size = cqUnit.getSize();
|
|||
|
|
+ long storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size);
|
|||
|
|
+ return storeTime;
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ return -1;
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java
|
|||
|
|
index 387c233bf..7108c835c 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java
|
|||
|
|
@@ -24,6 +24,7 @@ import java.util.Map;
|
|||
|
|
import java.util.concurrent.ConcurrentSkipListMap;
|
|||
|
|
import java.util.function.Function;
|
|||
|
|
import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+import org.apache.rocketmq.common.Pair;
|
|||
|
|
import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
import org.apache.rocketmq.common.attribute.CQType;
|
|||
|
|
import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
@@ -311,6 +312,11 @@ public class BatchConsumeQueue implements ConsumeQueueInterface {
|
|||
|
|
return new BatchConsumeQueueIterator(sbr);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
+ public ReferredIterator<CqUnit> iterateFrom(long startIndex, int count) {
|
|||
|
|
+ return iterateFrom(startIndex);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
@Override
|
|||
|
|
public CqUnit get(long offset) {
|
|||
|
|
ReferredIterator<CqUnit> it = iterateFrom(offset);
|
|||
|
|
@@ -320,6 +326,20 @@ public class BatchConsumeQueue implements ConsumeQueueInterface {
|
|||
|
|
return it.nextAndRelease();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
+ public Pair<CqUnit, Long> getCqUnitAndStoreTime(long index) {
|
|||
|
|
+ CqUnit cqUnit = get(index);
|
|||
|
|
+ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit);
|
|||
|
|
+ return new Pair<>(cqUnit, messageStoreTime);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public Pair<CqUnit, Long> getEarliestUnitAndStoreTime() {
|
|||
|
|
+ CqUnit cqUnit = getEarliestUnit();
|
|||
|
|
+ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit);
|
|||
|
|
+ return new Pair<>(cqUnit, messageStoreTime);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
@Override
|
|||
|
|
public CqUnit getEarliestUnit() {
|
|||
|
|
return get(minOffsetInQueue);
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java
|
|||
|
|
index 55d080829..c65f2a68b 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java
|
|||
|
|
@@ -18,10 +18,12 @@
|
|||
|
|
package org.apache.rocketmq.store.queue;
|
|||
|
|
|
|||
|
|
import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+import org.apache.rocketmq.common.Pair;
|
|||
|
|
import org.apache.rocketmq.common.attribute.CQType;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
import org.apache.rocketmq.store.MessageFilter;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
|
|||
|
|
public interface ConsumeQueueInterface extends FileQueueLifeCycle {
|
|||
|
|
/**
|
|||
|
|
@@ -44,6 +46,16 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle {
|
|||
|
|
*/
|
|||
|
|
ReferredIterator<CqUnit> iterateFrom(long startIndex);
|
|||
|
|
|
|||
|
|
+ /**
|
|||
|
|
+ * Get the units from the start offset.
|
|||
|
|
+ *
|
|||
|
|
+ * @param startIndex start index
|
|||
|
|
+ * @param count the unit counts will be iterated
|
|||
|
|
+ * @return the unit iterateFrom
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ ReferredIterator<CqUnit> iterateFrom(long startIndex, int count) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
/**
|
|||
|
|
* Get cq unit at specified index
|
|||
|
|
* @param index index
|
|||
|
|
@@ -51,6 +63,18 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle {
|
|||
|
|
*/
|
|||
|
|
CqUnit get(long index);
|
|||
|
|
|
|||
|
|
+ /**
|
|||
|
|
+ * Get earliest cq unit
|
|||
|
|
+ * @return the cq unit and message storeTime at index
|
|||
|
|
+ */
|
|||
|
|
+ Pair<CqUnit, Long> getCqUnitAndStoreTime(long index);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Get earliest cq unit
|
|||
|
|
+ * @return earliest cq unit and message storeTime
|
|||
|
|
+ */
|
|||
|
|
+ Pair<CqUnit, Long> getEarliestUnitAndStoreTime();
|
|||
|
|
+
|
|||
|
|
/**
|
|||
|
|
* Get earliest cq unit
|
|||
|
|
* @return earliest cq unit
|
|||
|
|
@@ -153,8 +177,9 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle {
|
|||
|
|
* Assign queue offset.
|
|||
|
|
* @param queueOffsetAssigner the delegated queue offset assigner
|
|||
|
|
* @param msg message itself
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
*/
|
|||
|
|
- void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg);
|
|||
|
|
+ void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg) throws RocksDBException;
|
|||
|
|
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
|
|||
|
|
index d03d15d65..616511b67 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
|
|||
|
|
@@ -16,64 +16,128 @@
|
|||
|
|
*/
|
|||
|
|
package org.apache.rocketmq.store.queue;
|
|||
|
|
|
|||
|
|
+import java.io.File;
|
|||
|
|
import java.nio.ByteBuffer;
|
|||
|
|
import java.util.ArrayList;
|
|||
|
|
+import java.util.Iterator;
|
|||
|
|
import java.util.List;
|
|||
|
|
+import java.util.Map;
|
|||
|
|
+import java.util.Map.Entry;
|
|||
|
|
+import java.util.Objects;
|
|||
|
|
+import java.util.Optional;
|
|||
|
|
import java.util.concurrent.BlockingQueue;
|
|||
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+import java.util.concurrent.ConcurrentMap;
|
|||
|
|
import java.util.concurrent.CountDownLatch;
|
|||
|
|
import java.util.concurrent.ExecutorService;
|
|||
|
|
import java.util.concurrent.FutureTask;
|
|||
|
|
import java.util.concurrent.LinkedBlockingQueue;
|
|||
|
|
import java.util.concurrent.TimeUnit;
|
|||
|
|
+import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
import org.apache.rocketmq.common.attribute.CQType;
|
|||
|
|
-import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageDecoder;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
import org.apache.rocketmq.common.utils.QueueTypeUtils;
|
|||
|
|
import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
-import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
import org.apache.rocketmq.store.CommitLog;
|
|||
|
|
import org.apache.rocketmq.store.ConsumeQueue;
|
|||
|
|
import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
-import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
import org.apache.rocketmq.store.SelectMappedBufferResult;
|
|||
|
|
-import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
-
|
|||
|
|
-import java.io.File;
|
|||
|
|
-import java.util.Iterator;
|
|||
|
|
-import java.util.Map;
|
|||
|
|
-import java.util.Objects;
|
|||
|
|
-import java.util.Optional;
|
|||
|
|
-import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
-import java.util.concurrent.ConcurrentMap;
|
|||
|
|
|
|||
|
|
import static java.lang.String.format;
|
|||
|
|
import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue;
|
|||
|
|
import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue;
|
|||
|
|
|
|||
|
|
-public class ConsumeQueueStore {
|
|||
|
|
- private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
|
|||
|
|
+public class ConsumeQueueStore extends AbstractConsumeQueueStore {
|
|||
|
|
|
|||
|
|
- protected final DefaultMessageStore messageStore;
|
|||
|
|
- protected final MessageStoreConfig messageStoreConfig;
|
|||
|
|
- protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator();
|
|||
|
|
- protected final ConcurrentMap<String/* topic */, ConcurrentMap<Integer/* queueId */, ConsumeQueueInterface>> consumeQueueTable;
|
|||
|
|
+ public ConsumeQueueStore(DefaultMessageStore messageStore) {
|
|||
|
|
+ super(messageStore);
|
|||
|
|
+ }
|
|||
|
|
|
|||
|
|
- public ConsumeQueueStore(DefaultMessageStore messageStore, MessageStoreConfig messageStoreConfig) {
|
|||
|
|
- this.messageStore = messageStore;
|
|||
|
|
- this.messageStoreConfig = messageStoreConfig;
|
|||
|
|
- this.consumeQueueTable = new ConcurrentHashMap<>(32);
|
|||
|
|
+ @Override
|
|||
|
|
+ public void start() {
|
|||
|
|
+ log.info("Default ConsumeQueueStore start!");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private FileQueueLifeCycle getLifeCycle(String topic, int queueId) {
|
|||
|
|
- return findOrCreateConsumeQueue(topic, queueId);
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean load() {
|
|||
|
|
+ boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ);
|
|||
|
|
+ boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ);
|
|||
|
|
+ return cqLoadResult && bcqLoadResult;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean loadAfterDestroy() {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void recover() {
|
|||
|
|
+ for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
+ for (ConsumeQueueInterface logic : maps.values()) {
|
|||
|
|
+ this.recover(logic);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean recoverConcurrently() {
|
|||
|
|
+ int count = 0;
|
|||
|
|
+ for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
+ count += maps.values().size();
|
|||
|
|
+ }
|
|||
|
|
+ final CountDownLatch countDownLatch = new CountDownLatch(count);
|
|||
|
|
+ BlockingQueue<Runnable> recoverQueue = new LinkedBlockingQueue<>();
|
|||
|
|
+ final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_");
|
|||
|
|
+ List<FutureTask<Boolean>> result = new ArrayList<>(count);
|
|||
|
|
+ try {
|
|||
|
|
+ for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
+ for (final ConsumeQueueInterface logic : maps.values()) {
|
|||
|
|
+ FutureTask<Boolean> futureTask = new FutureTask<>(() -> {
|
|||
|
|
+ boolean ret = true;
|
|||
|
|
+ try {
|
|||
|
|
+ logic.recover();
|
|||
|
|
+ } catch (Throwable e) {
|
|||
|
|
+ ret = false;
|
|||
|
|
+ log.error("Exception occurs while recover consume queue concurrently, " +
|
|||
|
|
+ "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e);
|
|||
|
|
+ } finally {
|
|||
|
|
+ countDownLatch.countDown();
|
|||
|
|
+ }
|
|||
|
|
+ return ret;
|
|||
|
|
+ });
|
|||
|
|
+
|
|||
|
|
+ result.add(futureTask);
|
|||
|
|
+ executor.submit(futureTask);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ countDownLatch.await();
|
|||
|
|
+ for (FutureTask<Boolean> task : result) {
|
|||
|
|
+ if (task != null && task.isDone()) {
|
|||
|
|
+ if (!task.get()) {
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ log.error("Exception occurs while recover consume queue concurrently", e);
|
|||
|
|
+ return false;
|
|||
|
|
+ } finally {
|
|||
|
|
+ executor.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean shutdown() {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
public long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset) {
|
|||
|
|
FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
|
|||
|
|
return fileQueueLifeCycle.rollNextFile(offset);
|
|||
|
|
@@ -83,32 +147,53 @@ public class ConsumeQueueStore {
|
|||
|
|
consumeQueue.correctMinOffset(minCommitLogOffset);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- /**
|
|||
|
|
- * Apply the dispatched request and build the consume queue. This function should be idempotent.
|
|||
|
|
- *
|
|||
|
|
- * @param consumeQueue consume queue
|
|||
|
|
- * @param request dispatch request
|
|||
|
|
- */
|
|||
|
|
- public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) {
|
|||
|
|
- consumeQueue.putMessagePositionInfoWrapper(request);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
+ @Override
|
|||
|
|
public void putMessagePositionInfoWrapper(DispatchRequest dispatchRequest) {
|
|||
|
|
ConsumeQueueInterface cq = this.findOrCreateConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId());
|
|||
|
|
this.putMessagePositionInfoWrapper(cq, dispatchRequest);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
+ public List<ByteBuffer> rangeQuery(String topic, int queueId, long startIndex, int num) {
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ByteBuffer get(String topic, int queueId, long startIndex) {
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMaxOffsetInQueue(String topic, int queueId) {
|
|||
|
|
+ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId);
|
|||
|
|
+ if (logic != null) {
|
|||
|
|
+ return logic.getMaxOffsetInQueue();
|
|||
|
|
+ }
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) {
|
|||
|
|
+ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId);
|
|||
|
|
+ if (logic != null) {
|
|||
|
|
+ long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType);
|
|||
|
|
+ // Make sure the result offset is in valid range.
|
|||
|
|
+ resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue());
|
|||
|
|
+ resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue());
|
|||
|
|
+ return resultOffset;
|
|||
|
|
+ }
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private FileQueueLifeCycle getLifeCycle(String topic, int queueId) {
|
|||
|
|
+ return findOrCreateConsumeQueue(topic, queueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
public boolean load(ConsumeQueueInterface consumeQueue) {
|
|||
|
|
FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
|
|||
|
|
return fileQueueLifeCycle.load();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public boolean load() {
|
|||
|
|
- boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ);
|
|||
|
|
- boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ);
|
|||
|
|
- return cqLoadResult && bcqLoadResult;
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
private boolean loadConsumeQueues(String storePath, CQType cqType) {
|
|||
|
|
File dirLogic = new File(storePath);
|
|||
|
|
File[] fileTopicList = dirLogic.listFiles();
|
|||
|
|
@@ -189,62 +274,17 @@ public class ConsumeQueueStore {
|
|||
|
|
fileQueueLifeCycle.recover();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public void recover() {
|
|||
|
|
- for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
- for (ConsumeQueueInterface logic : maps.values()) {
|
|||
|
|
- this.recover(logic);
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public boolean recoverConcurrently() {
|
|||
|
|
- int count = 0;
|
|||
|
|
- for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
- count += maps.values().size();
|
|||
|
|
- }
|
|||
|
|
- final CountDownLatch countDownLatch = new CountDownLatch(count);
|
|||
|
|
- BlockingQueue<Runnable> recoverQueue = new LinkedBlockingQueue<>();
|
|||
|
|
- final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_");
|
|||
|
|
- List<FutureTask<Boolean>> result = new ArrayList<>(count);
|
|||
|
|
- try {
|
|||
|
|
- for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
- for (final ConsumeQueueInterface logic : maps.values()) {
|
|||
|
|
- FutureTask<Boolean> futureTask = new FutureTask<>(() -> {
|
|||
|
|
- boolean ret = true;
|
|||
|
|
- try {
|
|||
|
|
- logic.recover();
|
|||
|
|
- } catch (Throwable e) {
|
|||
|
|
- ret = false;
|
|||
|
|
- log.error("Exception occurs while recover consume queue concurrently, " +
|
|||
|
|
- "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e);
|
|||
|
|
- } finally {
|
|||
|
|
- countDownLatch.countDown();
|
|||
|
|
- }
|
|||
|
|
- return ret;
|
|||
|
|
- });
|
|||
|
|
-
|
|||
|
|
- result.add(futureTask);
|
|||
|
|
- executor.submit(futureTask);
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
- countDownLatch.await();
|
|||
|
|
- for (FutureTask<Boolean> task : result) {
|
|||
|
|
- if (task != null && task.isDone()) {
|
|||
|
|
- if (!task.get()) {
|
|||
|
|
- return false;
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
- }
|
|||
|
|
- } catch (Exception e) {
|
|||
|
|
- log.error("Exception occurs while recover consume queue concurrently", e);
|
|||
|
|
- return false;
|
|||
|
|
- } finally {
|
|||
|
|
- executor.shutdown();
|
|||
|
|
+ @Override
|
|||
|
|
+ public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) {
|
|||
|
|
+ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId);
|
|||
|
|
+ if (logic != null) {
|
|||
|
|
+ return logic.getMaxPhysicOffset();
|
|||
|
|
}
|
|||
|
|
- return true;
|
|||
|
|
+ return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public long getMaxOffsetInConsumeQueue() {
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMaxPhyOffsetInConsumeQueue() {
|
|||
|
|
long maxPhysicOffset = -1L;
|
|||
|
|
for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
for (ConsumeQueueInterface logic : maps.values()) {
|
|||
|
|
@@ -256,11 +296,22 @@ public class ConsumeQueueStore {
|
|||
|
|
return maxPhysicOffset;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMinOffsetInQueue(String topic, int queueId) {
|
|||
|
|
+ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId);
|
|||
|
|
+ if (logic != null) {
|
|||
|
|
+ return logic.getMinOffsetInQueue();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return -1;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
public void checkSelf(ConsumeQueueInterface consumeQueue) {
|
|||
|
|
FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
|
|||
|
|
fileQueueLifeCycle.checkSelf();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public void checkSelf() {
|
|||
|
|
for (Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> topicEntry : this.consumeQueueTable.entrySet()) {
|
|||
|
|
for (Map.Entry<Integer, ConsumeQueueInterface> cqEntry : topicEntry.getValue().entrySet()) {
|
|||
|
|
@@ -269,16 +320,19 @@ public class ConsumeQueueStore {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) {
|
|||
|
|
FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
|
|||
|
|
return fileQueueLifeCycle.flush(flushLeastPages);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public void destroy(ConsumeQueueInterface consumeQueue) {
|
|||
|
|
FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
|
|||
|
|
fileQueueLifeCycle.destroy();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) {
|
|||
|
|
FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
|
|||
|
|
return fileQueueLifeCycle.deleteExpiredFile(minCommitLogPos);
|
|||
|
|
@@ -300,21 +354,20 @@ public class ConsumeQueueStore {
|
|||
|
|
fileQueueLifeCycle.cleanSwappedMap(forceCleanSwapIntervalMs);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) {
|
|||
|
|
FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
|
|||
|
|
return fileQueueLifeCycle.isFirstFileAvailable();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) {
|
|||
|
|
FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
|
|||
|
|
return fileQueueLifeCycle.isFirstFileExist();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) {
|
|||
|
|
- return doFindOrCreateConsumeQueue(topic, queueId);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- private ConsumeQueueInterface doFindOrCreateConsumeQueue(String topic, int queueId) {
|
|||
|
|
ConcurrentMap<Integer, ConsumeQueueInterface> map = consumeQueueTable.get(topic);
|
|||
|
|
if (null == map) {
|
|||
|
|
ConcurrentMap<Integer, ConsumeQueueInterface> newMap = new ConcurrentHashMap<>(128);
|
|||
|
|
@@ -361,46 +414,15 @@ public class ConsumeQueueStore {
|
|||
|
|
return logic;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public Long getMaxOffset(String topic, int queueId) {
|
|||
|
|
- return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public void setTopicQueueTable(ConcurrentMap<String, Long> topicQueueTable) {
|
|||
|
|
- this.queueOffsetOperator.setTopicQueueTable(topicQueueTable);
|
|||
|
|
- this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public ConcurrentMap getTopicQueueTable() {
|
|||
|
|
- return this.queueOffsetOperator.getTopicQueueTable();
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
public void setBatchTopicQueueTable(ConcurrentMap<String, Long> batchTopicQueueTable) {
|
|||
|
|
this.queueOffsetOperator.setBatchTopicQueueTable(batchTopicQueueTable);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public void assignQueueOffset(MessageExtBrokerInner msg) {
|
|||
|
|
- ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId());
|
|||
|
|
- consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) {
|
|||
|
|
- ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId());
|
|||
|
|
- consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
public void updateQueueOffset(String topic, int queueId, long offset) {
|
|||
|
|
String topicQueueKey = topic + "-" + queueId;
|
|||
|
|
this.queueOffsetOperator.updateQueueOffset(topicQueueKey, offset);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public void removeTopicQueueTable(String topic, Integer queueId) {
|
|||
|
|
- this.queueOffsetOperator.remove(topic, queueId);
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
- public ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> getConsumeQueueTable() {
|
|||
|
|
- return consumeQueueTable;
|
|||
|
|
- }
|
|||
|
|
-
|
|||
|
|
private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueueInterface consumeQueue) {
|
|||
|
|
ConcurrentMap<Integer/* queueId */, ConsumeQueueInterface> map = this.consumeQueueTable.get(topic);
|
|||
|
|
if (null == map) {
|
|||
|
|
@@ -412,6 +434,7 @@ public class ConsumeQueueStore {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public void recoverOffsetTable(long minPhyOffset) {
|
|||
|
|
ConcurrentMap<String, Long> cqOffsetTable = new ConcurrentHashMap<>(1024);
|
|||
|
|
ConcurrentMap<String, Long> bcqOffsetTable = new ConcurrentHashMap<>(1024);
|
|||
|
|
@@ -431,7 +454,7 @@ public class ConsumeQueueStore {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- //Correct unSubmit consumeOffset
|
|||
|
|
+ // Correct unSubmit consumeOffset
|
|||
|
|
if (messageStoreConfig.isDuplicationEnable()) {
|
|||
|
|
SelectMappedBufferResult lastBuffer = null;
|
|||
|
|
long startReadOffset = messageStore.getCommitLog().getConfirmOffset() == -1 ? 0 : messageStore.getCommitLog().getConfirmOffset();
|
|||
|
|
@@ -476,6 +499,7 @@ public class ConsumeQueueStore {
|
|||
|
|
this.setBatchTopicQueueTable(bcqOffsetTable);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public void destroy() {
|
|||
|
|
for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
for (ConsumeQueueInterface logic : maps.values()) {
|
|||
|
|
@@ -484,8 +508,9 @@ public class ConsumeQueueStore {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public void cleanExpired(long minCommitLogOffset) {
|
|||
|
|
- Iterator<Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>>> it = this.consumeQueueTable.entrySet().iterator();
|
|||
|
|
+ Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>>> it = this.consumeQueueTable.entrySet().iterator();
|
|||
|
|
while (it.hasNext()) {
|
|||
|
|
Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> next = it.next();
|
|||
|
|
String topic = next.getKey();
|
|||
|
|
@@ -526,14 +551,16 @@ public class ConsumeQueueStore {
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- public void truncateDirty(long phyOffset) {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void truncateDirty(long offsetToTruncate) {
|
|||
|
|
for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
for (ConsumeQueueInterface logic : maps.values()) {
|
|||
|
|
- this.truncateDirtyLogicFiles(logic, phyOffset);
|
|||
|
|
+ this.truncateDirtyLogicFiles(logic, offsetToTruncate);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ @Override
|
|||
|
|
public long getTotalSize() {
|
|||
|
|
long totalSize = 0;
|
|||
|
|
for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..268803dcc
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java
|
|||
|
|
@@ -0,0 +1,289 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.queue;
|
|||
|
|
+
|
|||
|
|
+import java.nio.ByteBuffer;
|
|||
|
|
+import java.util.List;
|
|||
|
|
+import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
+import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+
|
|||
|
|
+public interface ConsumeQueueStoreInterface {
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Start the consumeQueueStore
|
|||
|
|
+ */
|
|||
|
|
+ void start();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Load from file.
|
|||
|
|
+ * @return true if loaded successfully.
|
|||
|
|
+ */
|
|||
|
|
+ boolean load();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * load after destroy
|
|||
|
|
+ */
|
|||
|
|
+ boolean loadAfterDestroy();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Recover from file.
|
|||
|
|
+ */
|
|||
|
|
+ void recover();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Recover concurrently from file.
|
|||
|
|
+ * @return true if recovered successfully.
|
|||
|
|
+ */
|
|||
|
|
+ boolean recoverConcurrently();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Shutdown the consumeQueueStore
|
|||
|
|
+ * @return true if shutdown successfully.
|
|||
|
|
+ */
|
|||
|
|
+ boolean shutdown();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * destroy all consumeQueues
|
|||
|
|
+ */
|
|||
|
|
+ void destroy();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * destroy the specific consumeQueue
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Flush cache to file.
|
|||
|
|
+ * @param consumeQueue the consumeQueue will be flushed
|
|||
|
|
+ * @param flushLeastPages the minimum number of pages to be flushed
|
|||
|
|
+ * @return true if any data has been flushed.
|
|||
|
|
+ */
|
|||
|
|
+ boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * clean expired data from minPhyOffset
|
|||
|
|
+ * @param minPhyOffset
|
|||
|
|
+ */
|
|||
|
|
+ void cleanExpired(long minPhyOffset);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Check files.
|
|||
|
|
+ */
|
|||
|
|
+ void checkSelf();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Delete expired files ending at min commit log position.
|
|||
|
|
+ * @param consumeQueue
|
|||
|
|
+ * @param minCommitLogPos min commit log position
|
|||
|
|
+ * @return deleted file numbers.
|
|||
|
|
+ */
|
|||
|
|
+ int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Is the first file available?
|
|||
|
|
+ * @param consumeQueue
|
|||
|
|
+ * @return true if it's available
|
|||
|
|
+ */
|
|||
|
|
+ boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Does the first file exist?
|
|||
|
|
+ * @param consumeQueue
|
|||
|
|
+ * @return true if it exists
|
|||
|
|
+ */
|
|||
|
|
+ boolean isFirstFileExist(ConsumeQueueInterface consumeQueue);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Roll to next file.
|
|||
|
|
+ * @param consumeQueue
|
|||
|
|
+ * @param offset next beginning offset
|
|||
|
|
+ * @return the beginning offset of the next file
|
|||
|
|
+ */
|
|||
|
|
+ long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * truncate dirty data
|
|||
|
|
+ * @param offsetToTruncate
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ void truncateDirty(long offsetToTruncate) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Apply the dispatched request and build the consume queue. This function should be idempotent.
|
|||
|
|
+ *
|
|||
|
|
+ * @param consumeQueue consume queue
|
|||
|
|
+ * @param request dispatch request
|
|||
|
|
+ */
|
|||
|
|
+ void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Apply the dispatched request. This function should be idempotent.
|
|||
|
|
+ *
|
|||
|
|
+ * @param request dispatch request
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode will throw exception
|
|||
|
|
+ */
|
|||
|
|
+ void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * range query cqUnit(ByteBuffer) in rocksdb
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ * @param startIndex
|
|||
|
|
+ * @param num
|
|||
|
|
+ * @return the byteBuffer list of the topic-queueId in rocksdb
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * query cqUnit(ByteBuffer) in rocksdb
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ * @param startIndex
|
|||
|
|
+ * @return the byteBuffer of the topic-queueId in rocksdb
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ ByteBuffer get(final String topic, final int queueId, final long startIndex) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * get consumeQueue table
|
|||
|
|
+ * @return the consumeQueue table
|
|||
|
|
+ */
|
|||
|
|
+ ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> getConsumeQueueTable();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Assign queue offset.
|
|||
|
|
+ * @param msg message itself
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Increase queue offset.
|
|||
|
|
+ * @param msg message itself
|
|||
|
|
+ * @param messageNum message number
|
|||
|
|
+ */
|
|||
|
|
+ void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * recover topicQueue table by minPhyOffset
|
|||
|
|
+ * @param minPhyOffset
|
|||
|
|
+ */
|
|||
|
|
+ void recoverOffsetTable(long minPhyOffset);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * set topicQueue table
|
|||
|
|
+ * @param topicQueueTable
|
|||
|
|
+ */
|
|||
|
|
+ void setTopicQueueTable(ConcurrentMap<String, Long> topicQueueTable);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * remove topic-queueId from topicQueue table
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ */
|
|||
|
|
+ void removeTopicQueueTable(String topic, Integer queueId);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * get topicQueue table
|
|||
|
|
+ * @return the topicQueue table
|
|||
|
|
+ */
|
|||
|
|
+ ConcurrentMap getTopicQueueTable();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * get the max physical offset in consumeQueue
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ * @return
|
|||
|
|
+ */
|
|||
|
|
+ Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * get maxOffset of specific topic-queueId in topicQueue table
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ * @return the max offset in QueueOffsetOperator
|
|||
|
|
+ */
|
|||
|
|
+ Long getMaxOffset(String topic, int queueId);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * get max physic offset in consumeQueue
|
|||
|
|
+ * @return the max physic offset in consumeQueue
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ long getMaxPhyOffsetInConsumeQueue() throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * get min logic offset of specific topic-queueId in consumeQueue
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ * @return the min logic offset of specific topic-queueId in consumeQueue
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ long getMinOffsetInQueue(final String topic, final int queueId) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * get max logic offset of specific topic-queueId in consumeQueue
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ * @return the max logic offset of specific topic-queueId in consumeQueue
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ long getMaxOffsetInQueue(final String topic, final int queueId) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Get the message whose timestamp is the smallest, greater than or equal to the given time and when there are more
|
|||
|
|
+ * than one message satisfy the condition, decide which one to return based on boundaryType.
|
|||
|
|
+ * @param timestamp timestamp
|
|||
|
|
+ * @param boundaryType Lower or Upper
|
|||
|
|
+ * @return the offset(index)
|
|||
|
|
+ * @throws RocksDBException only in rocksdb mode
|
|||
|
|
+ */
|
|||
|
|
+ long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * find or create the consumeQueue
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ * @return the consumeQueue
|
|||
|
|
+ */
|
|||
|
|
+ ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * find the consumeQueueMap of topic
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @return the consumeQueueMap of topic
|
|||
|
|
+ */
|
|||
|
|
+ ConcurrentMap<Integer, ConsumeQueueInterface> findConsumeQueueMap(String topic);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * get the total size of all consumeQueue
|
|||
|
|
+ * @return the total size of all consumeQueue
|
|||
|
|
+ */
|
|||
|
|
+ long getTotalSize();
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Get store time from commitlog by cqUnit
|
|||
|
|
+ * @param cqUnit
|
|||
|
|
+ * @return
|
|||
|
|
+ */
|
|||
|
|
+ long getStoreTime(CqUnit cqUnit);
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..d6291d908
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java
|
|||
|
|
@@ -0,0 +1,76 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.queue;
|
|||
|
|
+
|
|||
|
|
+import java.util.ArrayList;
|
|||
|
|
+import java.util.List;
|
|||
|
|
+import java.util.Map;
|
|||
|
|
+
|
|||
|
|
+import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageConst;
|
|||
|
|
+import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
+import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
+import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
+
|
|||
|
|
+public class MultiDispatch {
|
|||
|
|
+
|
|||
|
|
+ public static String lmqQueueKey(String queueName) {
|
|||
|
|
+ StringBuilder keyBuilder = new StringBuilder();
|
|||
|
|
+ keyBuilder.append(queueName);
|
|||
|
|
+ keyBuilder.append('-');
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ keyBuilder.append(queueId);
|
|||
|
|
+ return keyBuilder.toString();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static boolean isNeedHandleMultiDispatch(MessageStoreConfig messageStoreConfig, String topic) {
|
|||
|
|
+ return messageStoreConfig.isEnableMultiDispatch()
|
|||
|
|
+ && !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)
|
|||
|
|
+ && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX)
|
|||
|
|
+ && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static boolean checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, DispatchRequest dispatchRequest) {
|
|||
|
|
+ if (!isNeedHandleMultiDispatch(messageStoreConfig, dispatchRequest.getTopic())) {
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+ Map<String, String> prop = dispatchRequest.getPropertiesMap();
|
|||
|
|
+ if (prop == null || prop.isEmpty()) {
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+ String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
|
|||
|
|
+ String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);
|
|||
|
|
+ if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) {
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static List<DispatchRequest> checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, List<DispatchRequest> dispatchRequests) {
|
|||
|
|
+ if (!messageStoreConfig.isEnableMultiDispatch() || dispatchRequests == null || dispatchRequests.size() == 0) {
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ List<DispatchRequest> result = new ArrayList<>();
|
|||
|
|
+ for (DispatchRequest dispatchRequest : dispatchRequests) {
|
|||
|
|
+ if (checkMultiDispatchQueue(messageStoreConfig, dispatchRequest)) {
|
|||
|
|
+ result.add(dispatchRequest);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ return dispatchRequests;
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java
|
|||
|
|
index 2545bbf52..8da374828 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java
|
|||
|
|
@@ -41,6 +41,10 @@ public class QueueOffsetOperator {
|
|||
|
|
return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ public Long getTopicQueueNextOffset(String topicQueueKey) {
|
|||
|
|
+ return this.topicQueueTable.get(topicQueueKey);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
public void increaseQueueOffset(String topicQueueKey, short messageNum) {
|
|||
|
|
Long queueOffset = ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L);
|
|||
|
|
topicQueueTable.put(topicQueueKey, queueOffset + messageNum);
|
|||
|
|
@@ -63,6 +67,10 @@ public class QueueOffsetOperator {
|
|||
|
|
return ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
+ public Long getLmqTopicQueueNextOffset(String topicQueueKey) {
|
|||
|
|
+ return this.lmqTopicQueueTable.get(topicQueueKey);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
public void increaseLmqOffset(String topicQueueKey, short messageNum) {
|
|||
|
|
Long lmqOffset = ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L);
|
|||
|
|
this.lmqTopicQueueTable.put(topicQueueKey, lmqOffset + messageNum);
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..759be395d
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java
|
|||
|
|
@@ -0,0 +1,437 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.queue;
|
|||
|
|
+
|
|||
|
|
+import java.nio.ByteBuffer;
|
|||
|
|
+import java.util.List;
|
|||
|
|
+
|
|||
|
|
+import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+import org.apache.rocketmq.common.Pair;
|
|||
|
|
+import org.apache.rocketmq.common.attribute.CQType;
|
|||
|
|
+import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageAccessor;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageConst;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+import org.apache.rocketmq.store.ConsumeQueue;
|
|||
|
|
+import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
+import org.apache.rocketmq.store.MessageFilter;
|
|||
|
|
+import org.apache.rocketmq.store.MessageStore;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+
|
|||
|
|
+public class RocksDBConsumeQueue implements ConsumeQueueInterface {
|
|||
|
|
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
|
|||
|
|
+ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
+ private final MessageStore messageStore;
|
|||
|
|
+ private final String topic;
|
|||
|
|
+ private final int queueId;
|
|||
|
|
+
|
|||
|
|
+ public RocksDBConsumeQueue(final MessageStore messageStore, final String topic, final int queueId) {
|
|||
|
|
+ this.messageStore = messageStore;
|
|||
|
|
+ this.topic = topic;
|
|||
|
|
+ this.queueId = queueId;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public RocksDBConsumeQueue(final String topic, final int queueId) {
|
|||
|
|
+ this.messageStore = null;
|
|||
|
|
+ this.topic = topic;
|
|||
|
|
+ this.queueId = queueId;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean load() {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void recover() {
|
|||
|
|
+ // ignore
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void checkSelf() {
|
|||
|
|
+ // ignore
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean flush(final int flushLeastPages) {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void destroy() {
|
|||
|
|
+ // ignore
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void truncateDirtyLogicFiles(long maxCommitLogPos) {
|
|||
|
|
+ // ignored
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public int deleteExpiredFile(long minCommitLogPos) {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long rollNextFile(long nextBeginOffset) {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean isFirstFileAvailable() {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean isFirstFileExist() {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) {
|
|||
|
|
+ // ignore
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void cleanSwappedMap(long forceCleanSwapIntervalMs) {
|
|||
|
|
+ // ignore
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMaxOffsetInQueue() {
|
|||
|
|
+ try {
|
|||
|
|
+ return this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("getMaxOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e);
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMessageTotalInQueue() {
|
|||
|
|
+ try {
|
|||
|
|
+ long maxOffsetInQueue = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId);
|
|||
|
|
+ long minOffsetInQueue = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId);
|
|||
|
|
+ return maxOffsetInQueue - minOffsetInQueue;
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("getMessageTotalInQueue Failed. topic: {}, queueId: {}, {}", topic, queueId, e);
|
|||
|
|
+ }
|
|||
|
|
+ return -1;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * We already implement it in RocksDBConsumeQueueStore.
|
|||
|
|
+ * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime
|
|||
|
|
+ * @param timestamp timestamp
|
|||
|
|
+ * @return
|
|||
|
|
+ */
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getOffsetInQueueByTime(long timestamp) {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * We already implement it in RocksDBConsumeQueueStore.
|
|||
|
|
+ * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime
|
|||
|
|
+ * @param timestamp timestamp
|
|||
|
|
+ * @param boundaryType Lower or Upper
|
|||
|
|
+ * @return
|
|||
|
|
+ */
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getOffsetInQueueByTime(long timestamp, BoundaryType boundaryType) {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMaxPhysicOffset() {
|
|||
|
|
+ Long maxPhyOffset = this.messageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue(topic, queueId);
|
|||
|
|
+ return maxPhyOffset == null ? -1 : maxPhyOffset;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMinLogicOffset() {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CQType getCQType() {
|
|||
|
|
+ return CQType.RocksDBCQ;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getTotalSize() {
|
|||
|
|
+ // ignored
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public int getUnitSize() {
|
|||
|
|
+ // attention: unitSize should equal to 'ConsumeQueue.CQ_STORE_UNIT_SIZE'
|
|||
|
|
+ return ConsumeQueue.CQ_STORE_UNIT_SIZE;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Ignored, we already implement this method
|
|||
|
|
+ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int)
|
|||
|
|
+ */
|
|||
|
|
+ @Override
|
|||
|
|
+ public void correctMinOffset(long minCommitLogOffset) {
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Ignored, in rocksdb mode, we build cq in RocksDBConsumeQueueStore
|
|||
|
|
+ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore#putMessagePosition()
|
|||
|
|
+ */
|
|||
|
|
+ @Override
|
|||
|
|
+ public void putMessagePositionInfoWrapper(DispatchRequest request) {
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg) throws RocksDBException {
|
|||
|
|
+ String topicQueueKey = getTopic() + "-" + getQueueId();
|
|||
|
|
+ Long queueOffset = queueOffsetOperator.getTopicQueueNextOffset(topicQueueKey);
|
|||
|
|
+ if (queueOffset == null) {
|
|||
|
|
+ // we will recover topic queue table from rocksdb when we use it.
|
|||
|
|
+ queueOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId);
|
|||
|
|
+ queueOffsetOperator.updateQueueOffset(topicQueueKey, queueOffset);
|
|||
|
|
+ }
|
|||
|
|
+ msg.setQueueOffset(queueOffset);
|
|||
|
|
+
|
|||
|
|
+ // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28),
|
|||
|
|
+ // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue.
|
|||
|
|
+ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
|
|||
|
|
+ if (StringUtils.isBlank(multiDispatchQueue)) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
|
|||
|
|
+ Long[] queueOffsets = new Long[queues.length];
|
|||
|
|
+ for (int i = 0; i < queues.length; i++) {
|
|||
|
|
+ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) {
|
|||
|
|
+ String key = MultiDispatch.lmqQueueKey(queues[i]);
|
|||
|
|
+ queueOffsets[i] = queueOffsetOperator.getLmqTopicQueueNextOffset(key);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET,
|
|||
|
|
+ StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER));
|
|||
|
|
+ msg.removeWaitStorePropertyString();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg, short messageNum) {
|
|||
|
|
+ String topicQueueKey = getTopic() + "-" + getQueueId();
|
|||
|
|
+ queueOffsetOperator.increaseQueueOffset(topicQueueKey, messageNum);
|
|||
|
|
+
|
|||
|
|
+ // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28),
|
|||
|
|
+ // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue.
|
|||
|
|
+ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
|
|||
|
|
+ if (StringUtils.isBlank(multiDispatchQueue)) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
|
|||
|
|
+ for (int i = 0; i < queues.length; i++) {
|
|||
|
|
+ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) {
|
|||
|
|
+ String key = MultiDispatch.lmqQueueKey(queues[i]);
|
|||
|
|
+ queueOffsetOperator.increaseLmqOffset(key, (short) 1);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long estimateMessageCount(long from, long to, MessageFilter filter) {
|
|||
|
|
+ // todo
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMinOffsetInQueue() {
|
|||
|
|
+ return this.messageStore.getMinOffsetInQueue(this.topic, this.queueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private int pullNum(long cqOffset, long maxCqOffset) {
|
|||
|
|
+ long diffLong = maxCqOffset - cqOffset;
|
|||
|
|
+ if (diffLong < Integer.MAX_VALUE) {
|
|||
|
|
+ int diffInt = (int) diffLong;
|
|||
|
|
+ return diffInt > 16 ? 16 : diffInt;
|
|||
|
|
+ }
|
|||
|
|
+ return 16;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ReferredIterator<CqUnit> iterateFrom(final long startIndex) {
|
|||
|
|
+ try {
|
|||
|
|
+ long maxCqOffset = getMaxOffsetInQueue();
|
|||
|
|
+ if (startIndex < maxCqOffset) {
|
|||
|
|
+ int num = pullNum(startIndex, maxCqOffset);
|
|||
|
|
+ return iterateFrom0(startIndex, num);
|
|||
|
|
+ }
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ log.error("[RocksDBConsumeQueue] iterateFrom error!", e);
|
|||
|
|
+ }
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ReferredIterator<CqUnit> iterateFrom(long startIndex, int count) throws RocksDBException {
|
|||
|
|
+ long maxCqOffset = getMaxOffsetInQueue();
|
|||
|
|
+ if (startIndex < maxCqOffset) {
|
|||
|
|
+ int num = Math.min((int)(maxCqOffset - startIndex), count);
|
|||
|
|
+ return iterateFrom0(startIndex, num);
|
|||
|
|
+ }
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CqUnit get(long index) {
|
|||
|
|
+ Pair<CqUnit, Long> pair = getCqUnitAndStoreTime(index);
|
|||
|
|
+ return pair == null ? null : pair.getObject1();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public Pair<CqUnit, Long> getCqUnitAndStoreTime(long index) {
|
|||
|
|
+ ByteBuffer byteBuffer;
|
|||
|
|
+ try {
|
|||
|
|
+ byteBuffer = this.messageStore.getQueueStore().get(topic, queueId, index);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("getUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e);
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ if (byteBuffer == null || byteBuffer.remaining() < RocksDBConsumeQueueTable.CQ_UNIT_SIZE) {
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ long phyOffset = byteBuffer.getLong();
|
|||
|
|
+ int size = byteBuffer.getInt();
|
|||
|
|
+ long tagCode = byteBuffer.getLong();
|
|||
|
|
+ long messageStoreTime = byteBuffer.getLong();
|
|||
|
|
+ return new Pair<>(new CqUnit(index, phyOffset, size, tagCode), messageStoreTime);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public Pair<CqUnit, Long> getEarliestUnitAndStoreTime() {
|
|||
|
|
+ try {
|
|||
|
|
+ long minOffset = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId);
|
|||
|
|
+ return getCqUnitAndStoreTime(minOffset);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("getEarliestUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e);
|
|||
|
|
+ }
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CqUnit getEarliestUnit() {
|
|||
|
|
+ Pair<CqUnit, Long> pair = getEarliestUnitAndStoreTime();
|
|||
|
|
+ return pair == null ? null : pair.getObject1();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CqUnit getLatestUnit() {
|
|||
|
|
+ try {
|
|||
|
|
+ long maxOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId);
|
|||
|
|
+ return get(maxOffset);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("getLatestUnit Failed. topic: {}, queueId: {}, {}", topic, queueId, e.getMessage());
|
|||
|
|
+ }
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getLastOffset() {
|
|||
|
|
+ return getMaxPhysicOffset();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ReferredIterator<CqUnit> iterateFrom0(final long startIndex, final int count) throws RocksDBException {
|
|||
|
|
+ List<ByteBuffer> byteBufferList = this.messageStore.getQueueStore().rangeQuery(topic, queueId, startIndex, count);
|
|||
|
|
+ if (byteBufferList == null || byteBufferList.isEmpty()) {
|
|||
|
|
+ if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
|
|||
|
|
+ log.warn("iterateFrom0 - find nothing, startIndex:{}, count:{}", startIndex, count);
|
|||
|
|
+ }
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ return new RocksDBConsumeQueueIterator(byteBufferList, startIndex);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String getTopic() {
|
|||
|
|
+ return topic;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public int getQueueId() {
|
|||
|
|
+ return queueId;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private class RocksDBConsumeQueueIterator implements ReferredIterator<CqUnit> {
|
|||
|
|
+ private final List<ByteBuffer> byteBufferList;
|
|||
|
|
+ private final long startIndex;
|
|||
|
|
+ private final int totalCount;
|
|||
|
|
+ private int currentIndex;
|
|||
|
|
+
|
|||
|
|
+ public RocksDBConsumeQueueIterator(final List<ByteBuffer> byteBufferList, final long startIndex) {
|
|||
|
|
+ this.byteBufferList = byteBufferList;
|
|||
|
|
+ this.startIndex = startIndex;
|
|||
|
|
+ this.totalCount = byteBufferList.size();
|
|||
|
|
+ this.currentIndex = 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean hasNext() {
|
|||
|
|
+ return this.currentIndex < this.totalCount;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CqUnit next() {
|
|||
|
|
+ if (!hasNext()) {
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+ final int currentIndex = this.currentIndex;
|
|||
|
|
+ final ByteBuffer byteBuffer = this.byteBufferList.get(currentIndex);
|
|||
|
|
+ CqUnit cqUnit = new CqUnit(this.startIndex + currentIndex, byteBuffer.getLong(), byteBuffer.getInt(), byteBuffer.getLong());
|
|||
|
|
+ this.currentIndex++;
|
|||
|
|
+ return cqUnit;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void remove() {
|
|||
|
|
+ throw new UnsupportedOperationException("remove");
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void release() {
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public CqUnit nextAndRelease() {
|
|||
|
|
+ try {
|
|||
|
|
+ return next();
|
|||
|
|
+ } finally {
|
|||
|
|
+ release();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..6fa66282e
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java
|
|||
|
|
@@ -0,0 +1,641 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.queue;
|
|||
|
|
+
|
|||
|
|
+import java.nio.ByteBuffer;
|
|||
|
|
+import java.util.HashMap;
|
|||
|
|
+import java.util.HashSet;
|
|||
|
|
+import java.util.Map;
|
|||
|
|
+import java.util.Set;
|
|||
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+
|
|||
|
|
+import org.apache.rocketmq.common.Pair;
|
|||
|
|
+import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
+import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+import org.apache.rocketmq.store.ConsumeQueue;
|
|||
|
|
+import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
+import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
+import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
|
|||
|
|
+import org.rocksdb.ColumnFamilyHandle;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+import org.rocksdb.RocksIterator;
|
|||
|
|
+import org.rocksdb.WriteBatch;
|
|||
|
|
+
|
|||
|
|
+import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8;
|
|||
|
|
+import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1;
|
|||
|
|
+
|
|||
|
|
+public class RocksDBConsumeQueueOffsetTable {
|
|||
|
|
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
|
|||
|
|
+ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);
|
|||
|
|
+ private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
+ private static final byte[] MAX_BYTES = "max".getBytes(CHARSET_UTF8);
|
|||
|
|
+ private static final byte[] MIN_BYTES = "min".getBytes(CHARSET_UTF8);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Rocksdb ConsumeQueue's Offset unit. Format:
|
|||
|
|
+ *
|
|||
|
|
+ * <pre>
|
|||
|
|
+ * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬─────────────┐
|
|||
|
|
+ * │ Topic Bytes Array Size │ CTRL_1 │ Topic Bytes Array │ CTRL_1 │ Max(Min) │ CTRL_1 │ QueueId │
|
|||
|
|
+ * │ (4 Bytes) │ (1 Bytes) │ (n Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │
|
|||
|
|
+ * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴─────────────┤
|
|||
|
|
+ * │ Key Unit │
|
|||
|
|
+ * │ │
|
|||
|
|
+ * </pre>
|
|||
|
|
+ *
|
|||
|
|
+ * <pre>
|
|||
|
|
+ * ┌─────────────────────────────┬────────────────────────┐
|
|||
|
|
+ * │ CommitLog Physical Offset │ ConsumeQueue Offset │
|
|||
|
|
+ * │ (8 Bytes) │ (8 Bytes) │
|
|||
|
|
+ * ├─────────────────────────────┴────────────────────────┤
|
|||
|
|
+ * │ Value Unit │
|
|||
|
|
+ * │ │
|
|||
|
|
+ * </pre>
|
|||
|
|
+ * ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) = 16 Bytes
|
|||
|
|
+ */
|
|||
|
|
+ private static final int OFFSET_PHY_OFFSET = 0;
|
|||
|
|
+ private static final int OFFSET_CQ_OFFSET = 8;
|
|||
|
|
+ /**
|
|||
|
|
+ *
|
|||
|
|
+ * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐
|
|||
|
|
+ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ Max(Min) │ CTRL_1 │ QueueId │
|
|||
|
|
+ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │
|
|||
|
|
+ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴─────────────┤
|
|||
|
|
+ */
|
|||
|
|
+ private static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4;
|
|||
|
|
+ private static final int OFFSET_VALUE_LENGTH = 8 + 8;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread.
|
|||
|
|
+ * @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread.
|
|||
|
|
+ * If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of
|
|||
|
|
+ * RocksDBConsumeQueueOffsetTable to find it.
|
|||
|
|
+ */
|
|||
|
|
+ private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC;
|
|||
|
|
+ private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(CHARSET_UTF8);
|
|||
|
|
+ private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length;
|
|||
|
|
+ private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN);
|
|||
|
|
+ private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN];
|
|||
|
|
+ private final ByteBuffer maxPhyOffsetBB;
|
|||
|
|
+ static {
|
|||
|
|
+ buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true);
|
|||
|
|
+ INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN);
|
|||
|
|
+ INNER_CHECKPOINT_TOPIC.get(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable;
|
|||
|
|
+ private final ConsumeQueueRocksDBStorage rocksDBStorage;
|
|||
|
|
+ private final DefaultMessageStore messageStore;
|
|||
|
|
+
|
|||
|
|
+ private ColumnFamilyHandle offsetCFH;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them
|
|||
|
|
+ * from heap to avoid accessing rocksdb.
|
|||
|
|
+ * @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset --> topicQueueMaxCqOffset
|
|||
|
|
+ * @see ConsumeQueue#getMinLogicOffset(), minLogicOffset --> topicQueueMinOffset
|
|||
|
|
+ */
|
|||
|
|
+ private final Map<String/* topic-queueId */, PhyAndCQOffset> topicQueueMinOffset;
|
|||
|
|
+ private final Map<String/* topic-queueId */, Long> topicQueueMaxCqOffset;
|
|||
|
|
+
|
|||
|
|
+ public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable,
|
|||
|
|
+ ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) {
|
|||
|
|
+ this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable;
|
|||
|
|
+ this.rocksDBStorage = rocksDBStorage;
|
|||
|
|
+ this.messageStore = messageStore;
|
|||
|
|
+ this.topicQueueMinOffset = new ConcurrentHashMap(1024);
|
|||
|
|
+ this.topicQueueMaxCqOffset = new ConcurrentHashMap(1024);
|
|||
|
|
+
|
|||
|
|
+ this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void load() {
|
|||
|
|
+ this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void updateTempTopicQueueMaxOffset(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
|
|||
|
|
+ final byte[] topicBytes, final DispatchRequest request,
|
|||
|
|
+ final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap) {
|
|||
|
|
+ buildOffsetKeyAndValueByteBuffer(offsetBBPair, topicBytes, request);
|
|||
|
|
+ ByteBuffer topicQueueId = offsetBBPair.getObject1();
|
|||
|
|
+ ByteBuffer maxOffsetBB = offsetBBPair.getObject2();
|
|||
|
|
+ Pair<ByteBuffer, DispatchRequest> old = tempTopicQueueMaxOffsetMap.get(topicQueueId);
|
|||
|
|
+ if (old == null) {
|
|||
|
|
+ tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair(maxOffsetBB, request));
|
|||
|
|
+ } else {
|
|||
|
|
+ long oldMaxOffset = old.getObject1().getLong(OFFSET_CQ_OFFSET);
|
|||
|
|
+ long maxOffset = maxOffsetBB.getLong(OFFSET_CQ_OFFSET);
|
|||
|
|
+ if (maxOffset >= oldMaxOffset) {
|
|||
|
|
+ ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void putMaxPhyAndCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap,
|
|||
|
|
+ final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException {
|
|||
|
|
+ for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
|
|||
|
|
+ writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ appendMaxPhyOffset(writeBatch, maxPhyOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void putHeapMaxCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap) {
|
|||
|
|
+ for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
|
|||
|
|
+ DispatchRequest request = entry.getValue().getObject2();
|
|||
|
|
+ putHeapMaxCqOffset(request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * When topic is deleted, we clean up its offset info in rocksdb.
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ * @throws RocksDBException
|
|||
|
|
+ */
|
|||
|
|
+ public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException {
|
|||
|
|
+ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
|
|||
|
|
+ final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false);
|
|||
|
|
+ byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array());
|
|||
|
|
+ Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null;
|
|||
|
|
+
|
|||
|
|
+ final ByteBuffer maxOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, true);
|
|||
|
|
+ byte[] maxOffsetBytes = this.rocksDBStorage.getOffset(maxOffsetKey.array());
|
|||
|
|
+ Long endCQOffset = (maxOffsetBytes != null) ? ByteBuffer.wrap(maxOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null;
|
|||
|
|
+
|
|||
|
|
+ writeBatch.delete(this.offsetCFH, minOffsetKey.array());
|
|||
|
|
+ writeBatch.delete(this.offsetCFH, maxOffsetKey.array());
|
|||
|
|
+
|
|||
|
|
+ String topicQueueId = buildTopicQueueId(topic, queueId);
|
|||
|
|
+ removeHeapMinCqOffset(topicQueueId);
|
|||
|
|
+ removeHeapMaxCqOffset(topicQueueId);
|
|||
|
|
+
|
|||
|
|
+ log.info("RocksDB offset table delete topic: {}, queueId: {}, minOffset: {}, maxOffset: {}", topic, queueId,
|
|||
|
|
+ startCQOffset, endCQOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void appendMaxPhyOffset(final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException {
|
|||
|
|
+ final ByteBuffer maxPhyOffsetBB = this.maxPhyOffsetBB;
|
|||
|
|
+ maxPhyOffsetBB.position(0).limit(8);
|
|||
|
|
+ maxPhyOffsetBB.putLong(maxPhyOffset);
|
|||
|
|
+ maxPhyOffsetBB.flip();
|
|||
|
|
+
|
|||
|
|
+ INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN);
|
|||
|
|
+ writeBatch.put(this.offsetCFH, INNER_CHECKPOINT_TOPIC, maxPhyOffsetBB);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long getMaxPhyOffset() throws RocksDBException {
|
|||
|
|
+ byte[] valueBytes = this.rocksDBStorage.getOffset(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY);
|
|||
|
|
+ if (valueBytes == null) {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+ ByteBuffer valueBB = ByteBuffer.wrap(valueBytes);
|
|||
|
|
+ return valueBB.getLong(0);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Traverse the offset table to find dirty topic
|
|||
|
|
+ * @param existTopicSet
|
|||
|
|
+ * @return
|
|||
|
|
+ */
|
|||
|
|
+ public Map<String, Set<Integer>> iterateOffsetTable2FindDirty(final Set<String> existTopicSet) {
|
|||
|
|
+ Map<String/* topic */, Set<Integer/* queueId */>> topicQueueIdToBeDeletedMap = new HashMap<>();
|
|||
|
|
+
|
|||
|
|
+ RocksIterator iterator = null;
|
|||
|
|
+ try {
|
|||
|
|
+ iterator = rocksDBStorage.seekOffsetCF();
|
|||
|
|
+ if (iterator == null) {
|
|||
|
|
+ return topicQueueIdToBeDeletedMap;
|
|||
|
|
+ }
|
|||
|
|
+ for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {
|
|||
|
|
+ byte[] key = iterator.key();
|
|||
|
|
+ byte[] value = iterator.value();
|
|||
|
|
+ if (key == null || key.length <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES
|
|||
|
|
+ || value == null || value.length != OFFSET_VALUE_LENGTH) {
|
|||
|
|
+ continue;
|
|||
|
|
+ }
|
|||
|
|
+ ByteBuffer keyBB = ByteBuffer.wrap(key);
|
|||
|
|
+ int topicLen = keyBB.getInt(0);
|
|||
|
|
+ byte[] topicBytes = new byte[topicLen];
|
|||
|
|
+ /**
|
|||
|
|
+ * "Topic Bytes Array Size" + "CTRL_1" = 4 + 1
|
|||
|
|
+ */
|
|||
|
|
+ keyBB.position(4 + 1);
|
|||
|
|
+ keyBB.get(topicBytes);
|
|||
|
|
+ String topic = new String(topicBytes, CHARSET_UTF8);
|
|||
|
|
+ if (TopicValidator.isSystemTopic(topic)) {
|
|||
|
|
+ continue;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * "Topic Bytes Array Size" + "CTRL_1" + "Topic Bytes Array" + "CTRL_1" + "Max(min)" + "CTRL_1"
|
|||
|
|
+ * = 4 + 1 + topicLen + 1 + 3 + 1
|
|||
|
|
+ */
|
|||
|
|
+ int queueId = keyBB.getInt(4 + 1 + topicLen + 1 + 3 + 1);
|
|||
|
|
+
|
|||
|
|
+ if (!existTopicSet.contains(topic)) {
|
|||
|
|
+ ByteBuffer valueBB = ByteBuffer.wrap(value);
|
|||
|
|
+ long cqOffset = valueBB.getLong(OFFSET_CQ_OFFSET);
|
|||
|
|
+
|
|||
|
|
+ Set<Integer> topicQueueIdSet = topicQueueIdToBeDeletedMap.get(topic);
|
|||
|
|
+ if (topicQueueIdSet == null) {
|
|||
|
|
+ Set<Integer> newSet = new HashSet<>();
|
|||
|
|
+ newSet.add(queueId);
|
|||
|
|
+ topicQueueIdToBeDeletedMap.put(topic, newSet);
|
|||
|
|
+ } else {
|
|||
|
|
+ topicQueueIdSet.add(queueId);
|
|||
|
|
+ }
|
|||
|
|
+ ERROR_LOG.info("RocksDBConsumeQueueOffsetTable has dirty cqOffset. topic: {}, queueId: {}, cqOffset: {}",
|
|||
|
|
+ topic, queueId, cqOffset);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ ERROR_LOG.error("iterateOffsetTable2MarkDirtyCQ Failed.", e);
|
|||
|
|
+ } finally {
|
|||
|
|
+ if (iterator != null) {
|
|||
|
|
+ iterator.close();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ return topicQueueIdToBeDeletedMap;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException {
|
|||
|
|
+ Long maxCqOffset = getHeapMaxCqOffset(topic, queueId);
|
|||
|
|
+
|
|||
|
|
+ if (maxCqOffset == null) {
|
|||
|
|
+ final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId);
|
|||
|
|
+ maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null;
|
|||
|
|
+ String topicQueueId = buildTopicQueueId(topic, queueId);
|
|||
|
|
+ this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, maxCqOffset != null ? maxCqOffset : -1L);
|
|||
|
|
+ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
|
|||
|
|
+ ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, maxCqOffset);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ return maxCqOffset;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * truncate dirty offset in rocksdb
|
|||
|
|
+ * @param offsetToTruncate
|
|||
|
|
+ * @throws RocksDBException
|
|||
|
|
+ */
|
|||
|
|
+ public void truncateDirty(long offsetToTruncate) throws RocksDBException {
|
|||
|
|
+ correctMaxPyhOffset(offsetToTruncate);
|
|||
|
|
+
|
|||
|
|
+ ConcurrentMap<String, TopicConfig> allTopicConfigMap = this.messageStore.getTopicConfigs();
|
|||
|
|
+ if (allTopicConfigMap == null) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ for (TopicConfig topicConfig : allTopicConfigMap.values()) {
|
|||
|
|
+ for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) {
|
|||
|
|
+ truncateDirtyOffset(topicConfig.getTopicName(), i);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private Pair<Boolean, Long> isMinOffsetOk(final String topic, final int queueId, final long minPhyOffset) throws RocksDBException {
|
|||
|
|
+ PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId);
|
|||
|
|
+ if (phyAndCQOffset != null) {
|
|||
|
|
+ final long phyOffset = phyAndCQOffset.getPhyOffset();
|
|||
|
|
+ final long cqOffset = phyAndCQOffset.getCqOffset();
|
|||
|
|
+
|
|||
|
|
+ return (phyOffset >= minPhyOffset) ? new Pair(true, cqOffset) : new Pair(false, cqOffset);
|
|||
|
|
+ }
|
|||
|
|
+ ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId);
|
|||
|
|
+ if (byteBuffer == null) {
|
|||
|
|
+ return new Pair(false, 0L);
|
|||
|
|
+ }
|
|||
|
|
+ final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET);
|
|||
|
|
+ final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET);
|
|||
|
|
+ if (phyOffset >= minPhyOffset) {
|
|||
|
|
+ String topicQueueId = buildTopicQueueId(topic, queueId);
|
|||
|
|
+ PhyAndCQOffset newPhyAndCQOffset = new PhyAndCQOffset(phyOffset, cqOffset);
|
|||
|
|
+ this.topicQueueMinOffset.putIfAbsent(topicQueueId, newPhyAndCQOffset);
|
|||
|
|
+ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
|
|||
|
|
+ ROCKSDB_LOG.warn("updateMinOffsetInQueue. {}, {}", topicQueueId, newPhyAndCQOffset);
|
|||
|
|
+ }
|
|||
|
|
+ return new Pair(true, cqOffset);
|
|||
|
|
+ }
|
|||
|
|
+ return new Pair(false, cqOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException {
|
|||
|
|
+ final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId);
|
|||
|
|
+ if (byteBuffer == null) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ long maxPhyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET);
|
|||
|
|
+ long maxCqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET);
|
|||
|
|
+ long maxPhyOffsetInCQ = getMaxPhyOffset();
|
|||
|
|
+
|
|||
|
|
+ if (maxPhyOffset >= maxPhyOffsetInCQ) {
|
|||
|
|
+ correctMaxCqOffset(topic, queueId, maxCqOffset, maxPhyOffsetInCQ);
|
|||
|
|
+ Long newMaxCqOffset = getHeapMaxCqOffset(topic, queueId);
|
|||
|
|
+ ROCKSDB_LOG.warn("truncateDirtyLogicFile topic: {}, queueId: {} from {} to {}", topic, queueId,
|
|||
|
|
+ maxPhyOffset, newMaxCqOffset);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void correctMaxPyhOffset(long maxPhyOffset) throws RocksDBException {
|
|||
|
|
+ if (!this.rocksDBStorage.hold()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ try {
|
|||
|
|
+ WriteBatch writeBatch = new WriteBatch();
|
|||
|
|
+ long oldMaxPhyOffset = getMaxPhyOffset();
|
|||
|
|
+ if (oldMaxPhyOffset <= maxPhyOffset) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ log.info("correctMaxPyhOffset, oldMaxPhyOffset={}, newMaxPhyOffset={}", oldMaxPhyOffset, maxPhyOffset);
|
|||
|
|
+ appendMaxPhyOffset(writeBatch, maxPhyOffset);
|
|||
|
|
+ this.rocksDBStorage.batchPut(writeBatch);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("correctMaxPyhOffset Failed.", e);
|
|||
|
|
+ throw e;
|
|||
|
|
+ } finally {
|
|||
|
|
+ this.rocksDBStorage.release();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long getMinCqOffset(String topic, int queueId) throws RocksDBException {
|
|||
|
|
+ final long minPhyOffset = this.messageStore.getMinPhyOffset();
|
|||
|
|
+ Pair<Boolean, Long> pair = isMinOffsetOk(topic, queueId, minPhyOffset);
|
|||
|
|
+ final long cqOffset = pair.getObject2();
|
|||
|
|
+ if (!pair.getObject1() && correctMinCqOffset(topic, queueId, cqOffset, minPhyOffset)) {
|
|||
|
|
+ PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId);
|
|||
|
|
+ if (phyAndCQOffset != null) {
|
|||
|
|
+ if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
|
|||
|
|
+ ROCKSDB_LOG.warn("getMinOffsetInQueue miss heap. topic: {}, queueId: {}, old: {}, new: {}",
|
|||
|
|
+ topic, queueId, cqOffset, phyAndCQOffset);
|
|||
|
|
+ }
|
|||
|
|
+ return phyAndCQOffset.getCqOffset();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ return cqOffset;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public Long getMaxPhyOffset(String topic, int queueId) {
|
|||
|
|
+ try {
|
|||
|
|
+ ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId);
|
|||
|
|
+ if (byteBuffer != null) {
|
|||
|
|
+ return byteBuffer.getLong(OFFSET_PHY_OFFSET);
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ ERROR_LOG.info("getMaxPhyOffset error. topic: {}, queueId: {}", topic, queueId);
|
|||
|
|
+ }
|
|||
|
|
+ return null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ByteBuffer getMinPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException {
|
|||
|
|
+ return getPhyAndCqOffsetInKV(topic, queueId, false);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ByteBuffer getMaxPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException {
|
|||
|
|
+ return getPhyAndCqOffsetInKV(topic, queueId, true);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException {
|
|||
|
|
+ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
|
|||
|
|
+ final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max);
|
|||
|
|
+
|
|||
|
|
+ byte[] value = this.rocksDBStorage.getOffset(keyBB.array());
|
|||
|
|
+ return (value != null) ? ByteBuffer.wrap(value) : null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private String buildTopicQueueId(final String topic, final int queueId) {
|
|||
|
|
+ return topic + "-" + queueId;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, final long minCQOffset) {
|
|||
|
|
+ String topicQueueId = buildTopicQueueId(topic, queueId);
|
|||
|
|
+ PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset);
|
|||
|
|
+ this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxCQOffset) {
|
|||
|
|
+ String topicQueueId = buildTopicQueueId(topic, queueId);
|
|||
|
|
+ Long oldMaxCqOffset = this.topicQueueMaxCqOffset.put(topicQueueId, maxCQOffset);
|
|||
|
|
+ if (oldMaxCqOffset != null && oldMaxCqOffset > maxCQOffset) {
|
|||
|
|
+ ERROR_LOG.error("cqOffset invalid0. old: {}, now: {}", oldMaxCqOffset, maxCQOffset);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private PhyAndCQOffset getHeapMinOffset(final String topic, final int queueId) {
|
|||
|
|
+ return this.topicQueueMinOffset.get(buildTopicQueueId(topic, queueId));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private Long getHeapMaxCqOffset(final String topic, final int queueId) {
|
|||
|
|
+ String topicQueueId = buildTopicQueueId(topic, queueId);
|
|||
|
|
+ return this.topicQueueMaxCqOffset.get(topicQueueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private PhyAndCQOffset removeHeapMinCqOffset(String topicQueueId) {
|
|||
|
|
+ return this.topicQueueMinOffset.remove(topicQueueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private Long removeHeapMaxCqOffset(String topicQueueId) {
|
|||
|
|
+ return this.topicQueueMaxCqOffset.remove(topicQueueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void updateCqOffset(final String topic, final int queueId, final long phyOffset,
|
|||
|
|
+ final long cqOffset, boolean max) throws RocksDBException {
|
|||
|
|
+ if (!this.rocksDBStorage.hold()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ WriteBatch writeBatch = new WriteBatch();
|
|||
|
|
+ try {
|
|||
|
|
+ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
|
|||
|
|
+ final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max);
|
|||
|
|
+
|
|||
|
|
+ final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset);
|
|||
|
|
+ writeBatch.put(this.offsetCFH, offsetKey.array(), offsetValue.array());
|
|||
|
|
+ this.rocksDBStorage.batchPut(writeBatch);
|
|||
|
|
+
|
|||
|
|
+ if (max) {
|
|||
|
|
+ putHeapMaxCqOffset(topic, queueId, cqOffset);
|
|||
|
|
+ } else {
|
|||
|
|
+ putHeapMinCqOffset(topic, queueId, phyOffset, cqOffset);
|
|||
|
|
+ }
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("updateCqOffset({}) failed.", max ? "max" : "min", e);
|
|||
|
|
+ throw e;
|
|||
|
|
+ } finally {
|
|||
|
|
+ writeBatch.close();
|
|||
|
|
+ this.rocksDBStorage.release();
|
|||
|
|
+ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
|
|||
|
|
+ ROCKSDB_LOG.warn("updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}",
|
|||
|
|
+ max ? "max" : "min", topic, queueId, phyOffset, cqOffset);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private boolean correctMaxCqOffset(final String topic, final int queueId, final long maxCQOffset,
|
|||
|
|
+ final long maxPhyOffsetInCQ) throws RocksDBException {
|
|||
|
|
+ // 'getMinOffsetInQueue' may correct minCqOffset and put it into heap
|
|||
|
|
+ long minCQOffset = getMinCqOffset(topic, queueId);
|
|||
|
|
+ PhyAndCQOffset minPhyAndCQOffset = getHeapMinOffset(topic, queueId);
|
|||
|
|
+ if (minPhyAndCQOffset == null
|
|||
|
|
+ || minPhyAndCQOffset.getCqOffset() != minCQOffset
|
|||
|
|
+ || minPhyAndCQOffset.getPhyOffset() > maxPhyOffsetInCQ) {
|
|||
|
|
+ ROCKSDB_LOG.info("[BUG] correctMaxCqOffset error! topic: {}, queueId: {}, maxPhyOffsetInCQ: {}, "
|
|||
|
|
+ + "minCqOffset: {}, phyAndCQOffset: {}",
|
|||
|
|
+ topic, queueId, maxPhyOffsetInCQ, minCQOffset, minPhyAndCQOffset);
|
|||
|
|
+ throw new RocksDBException("correctMaxCqOffset error");
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ long high = maxCQOffset;
|
|||
|
|
+ long low = minCQOffset;
|
|||
|
|
+ PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high,
|
|||
|
|
+ low, maxPhyOffsetInCQ, false);
|
|||
|
|
+
|
|||
|
|
+ long targetCQOffset = targetPhyAndCQOffset.getCqOffset();
|
|||
|
|
+ long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset();
|
|||
|
|
+
|
|||
|
|
+ if (targetCQOffset == -1) {
|
|||
|
|
+ if (maxCQOffset != minCQOffset) {
|
|||
|
|
+ updateCqOffset(topic, queueId, minPhyAndCQOffset.getPhyOffset(), minCQOffset, true);
|
|||
|
|
+ }
|
|||
|
|
+ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
|
|||
|
|
+ ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyAndCQOffset.getPhyOffset());
|
|||
|
|
+ }
|
|||
|
|
+ return false;
|
|||
|
|
+ } else {
|
|||
|
|
+ updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, true);
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private boolean correctMinCqOffset(final String topic, final int queueId,
|
|||
|
|
+ final long minCQOffset, final long minPhyOffset) throws RocksDBException {
|
|||
|
|
+ final ByteBuffer maxBB = getMaxPhyAndCqOffsetInKV(topic, queueId);
|
|||
|
|
+ if (maxBB == null) {
|
|||
|
|
+ updateCqOffset(topic, queueId, minPhyOffset, 0L, false);
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+ final long maxPhyOffset = maxBB.getLong(OFFSET_PHY_OFFSET);
|
|||
|
|
+ final long maxCQOffset = maxBB.getLong(OFFSET_CQ_OFFSET);
|
|||
|
|
+
|
|||
|
|
+ if (maxPhyOffset < minPhyOffset) {
|
|||
|
|
+ updateCqOffset(topic, queueId, minPhyOffset, maxCQOffset + 1, false);
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ long high = maxCQOffset;
|
|||
|
|
+ long low = minCQOffset;
|
|||
|
|
+ PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, low,
|
|||
|
|
+ minPhyOffset, true);
|
|||
|
|
+ long targetCQOffset = phyAndCQOffset.getCqOffset();
|
|||
|
|
+ long targetPhyOffset = phyAndCQOffset.getPhyOffset();
|
|||
|
|
+
|
|||
|
|
+ if (targetCQOffset == -1) {
|
|||
|
|
+ if (maxCQOffset != minCQOffset) {
|
|||
|
|
+ updateCqOffset(topic, queueId, maxPhyOffset, maxCQOffset, false);
|
|||
|
|
+ }
|
|||
|
|
+ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
|
|||
|
|
+ ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyOffset);
|
|||
|
|
+ }
|
|||
|
|
+ return false;
|
|||
|
|
+ } else {
|
|||
|
|
+ updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, false);
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static Pair<ByteBuffer, ByteBuffer> getOffsetByteBufferPair() {
|
|||
|
|
+ ByteBuffer offsetKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN);
|
|||
|
|
+ ByteBuffer offsetValue = ByteBuffer.allocateDirect(OFFSET_VALUE_LENGTH);
|
|||
|
|
+ return new Pair<>(offsetKey, offsetValue);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildOffsetKeyAndValueByteBuffer(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
|
|||
|
|
+ final byte[] topicBytes, final DispatchRequest request) {
|
|||
|
|
+ final ByteBuffer offsetKey = offsetBBPair.getObject1();
|
|||
|
|
+ buildOffsetKeyByteBuffer(offsetKey, topicBytes, request.getQueueId(), true);
|
|||
|
|
+
|
|||
|
|
+ final ByteBuffer offsetValue = offsetBBPair.getObject2();
|
|||
|
|
+ buildOffsetValueByteBuffer(offsetValue, request.getCommitLogOffset(), request.getConsumeQueueOffset());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) {
|
|||
|
|
+ ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);
|
|||
|
|
+ buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max);
|
|||
|
|
+ return byteBuffer;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) {
|
|||
|
|
+ byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);
|
|||
|
|
+ buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId,
|
|||
|
|
+ final boolean max) {
|
|||
|
|
+ byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1);
|
|||
|
|
+ if (max) {
|
|||
|
|
+ byteBuffer.put(MAX_BYTES);
|
|||
|
|
+ } else {
|
|||
|
|
+ byteBuffer.put(MIN_BYTES);
|
|||
|
|
+ }
|
|||
|
|
+ byteBuffer.put(CTRL_1).putInt(queueId);
|
|||
|
|
+ byteBuffer.flip();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) {
|
|||
|
|
+ byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH);
|
|||
|
|
+ buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) {
|
|||
|
|
+ final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH);
|
|||
|
|
+ buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset);
|
|||
|
|
+ return byteBuffer;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) {
|
|||
|
|
+ byteBuffer.putLong(phyOffset).putLong(cqOffset);
|
|||
|
|
+ byteBuffer.flip();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ static class PhyAndCQOffset {
|
|||
|
|
+ private final long phyOffset;
|
|||
|
|
+ private final long cqOffset;
|
|||
|
|
+
|
|||
|
|
+ public PhyAndCQOffset(final long phyOffset, final long cqOffset) {
|
|||
|
|
+ this.phyOffset = phyOffset;
|
|||
|
|
+ this.cqOffset = cqOffset;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long getPhyOffset() {
|
|||
|
|
+ return this.phyOffset;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long getCqOffset() {
|
|||
|
|
+ return this.cqOffset;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String toString() {
|
|||
|
|
+ return "[cqOffset=" + cqOffset + ", phyOffset=" + phyOffset + "]";
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..78456cfcd
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java
|
|||
|
|
@@ -0,0 +1,441 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.queue;
|
|||
|
|
+
|
|||
|
|
+import java.io.File;
|
|||
|
|
+import java.nio.ByteBuffer;
|
|||
|
|
+import java.util.ArrayList;
|
|||
|
|
+import java.util.HashMap;
|
|||
|
|
+import java.util.List;
|
|||
|
|
+import java.util.Map;
|
|||
|
|
+import java.util.Set;
|
|||
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+import java.util.concurrent.Executors;
|
|||
|
|
+import java.util.concurrent.ScheduledExecutorService;
|
|||
|
|
+import java.util.concurrent.TimeUnit;
|
|||
|
|
+
|
|||
|
|
+import org.apache.commons.io.FileUtils;
|
|||
|
|
+import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
+import org.apache.rocketmq.common.Pair;
|
|||
|
|
+import org.apache.rocketmq.common.ThreadFactoryImpl;
|
|||
|
|
+import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
+import org.apache.rocketmq.common.utils.DataConverter;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
+import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
+import org.apache.rocketmq.store.config.BrokerRole;
|
|||
|
|
+import org.apache.rocketmq.store.config.StorePathConfigHelper;
|
|||
|
|
+import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+import org.rocksdb.WriteBatch;
|
|||
|
|
+
|
|||
|
|
+public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
|
|||
|
|
+ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);
|
|||
|
|
+ private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
+ public static final byte CTRL_0 = '\u0000';
|
|||
|
|
+ public static final byte CTRL_1 = '\u0001';
|
|||
|
|
+ public static final byte CTRL_2 = '\u0002';
|
|||
|
|
+
|
|||
|
|
+ private static final int BATCH_SIZE = 16;
|
|||
|
|
+ public static final int MAX_KEY_LEN = 300;
|
|||
|
|
+
|
|||
|
|
+ private final ScheduledExecutorService scheduledExecutorService;
|
|||
|
|
+ private final String storePath;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * we use two tables with different ColumnFamilyHandle, called RocksDBConsumeQueueTable and RocksDBConsumeQueueOffsetTable.
|
|||
|
|
+ * 1.RocksDBConsumeQueueTable uses to store CqUnit[physicalOffset, msgSize, tagHashCode, msgStoreTime]
|
|||
|
|
+ * 2.RocksDBConsumeQueueOffsetTable uses to store physicalOffset and consumeQueueOffset(@see PhyAndCQOffset) of topic-queueId
|
|||
|
|
+ */
|
|||
|
|
+ private final ConsumeQueueRocksDBStorage rocksDBStorage;
|
|||
|
|
+ private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable;
|
|||
|
|
+ private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable;
|
|||
|
|
+
|
|||
|
|
+ private final WriteBatch writeBatch;
|
|||
|
|
+ private final List<DispatchRequest> bufferDRList;
|
|||
|
|
+ private final List<Pair<ByteBuffer, ByteBuffer>> cqBBPairList;
|
|||
|
|
+ private final List<Pair<ByteBuffer, ByteBuffer>> offsetBBPairList;
|
|||
|
|
+ private final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap;
|
|||
|
|
+ private volatile boolean isCQError = false;
|
|||
|
|
+
|
|||
|
|
+ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) {
|
|||
|
|
+ super(messageStore);
|
|||
|
|
+
|
|||
|
|
+ this.storePath = StorePathConfigHelper.getStorePathConsumeQueue(messageStoreConfig.getStorePathRootDir());
|
|||
|
|
+ this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath, 4);
|
|||
|
|
+ this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore);
|
|||
|
|
+ this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore);
|
|||
|
|
+
|
|||
|
|
+ this.writeBatch = new WriteBatch();
|
|||
|
|
+ this.bufferDRList = new ArrayList(BATCH_SIZE);
|
|||
|
|
+ this.cqBBPairList = new ArrayList(BATCH_SIZE);
|
|||
|
|
+ this.offsetBBPairList = new ArrayList(BATCH_SIZE);
|
|||
|
|
+ for (int i = 0; i < BATCH_SIZE; i++) {
|
|||
|
|
+ this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair());
|
|||
|
|
+ this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ this.tempTopicQueueMaxOffsetMap = new HashMap<>();
|
|||
|
|
+ this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(
|
|||
|
|
+ new ThreadFactoryImpl("RocksDBConsumeQueueStoreScheduledThread", messageStore.getBrokerIdentity()));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void start() {
|
|||
|
|
+ log.info("RocksDB ConsumeQueueStore start!");
|
|||
|
|
+ this.scheduledExecutorService.scheduleAtFixedRate(() -> {
|
|||
|
|
+ this.rocksDBStorage.statRocksdb(ROCKSDB_LOG);
|
|||
|
|
+ }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS);
|
|||
|
|
+
|
|||
|
|
+ this.scheduledExecutorService.scheduleWithFixedDelay(() -> {
|
|||
|
|
+ cleanDirty(messageStore.getTopicConfigs().keySet());
|
|||
|
|
+ }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void cleanDirty(final Set<String> existTopicSet) {
|
|||
|
|
+ try {
|
|||
|
|
+ Map<String, Set<Integer>> topicQueueIdToBeDeletedMap =
|
|||
|
|
+ this.rocksDBConsumeQueueOffsetTable.iterateOffsetTable2FindDirty(existTopicSet);
|
|||
|
|
+
|
|||
|
|
+ for (Map.Entry<String, Set<Integer>> entry : topicQueueIdToBeDeletedMap.entrySet()) {
|
|||
|
|
+ String topic = entry.getKey();
|
|||
|
|
+ for (int queueId : entry.getValue()) {
|
|||
|
|
+ destroy(new RocksDBConsumeQueue(topic, queueId));
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ log.error("cleanUnusedTopic Failed.", e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean load() {
|
|||
|
|
+ boolean result = this.rocksDBStorage.start();
|
|||
|
|
+ this.rocksDBConsumeQueueTable.load();
|
|||
|
|
+ this.rocksDBConsumeQueueOffsetTable.load();
|
|||
|
|
+ log.info("load rocksdb consume queue {}.", result ? "OK" : "Failed");
|
|||
|
|
+ return result;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean loadAfterDestroy() {
|
|||
|
|
+ return this.load();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void recover() {
|
|||
|
|
+ // ignored
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean recoverConcurrently() {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean shutdown() {
|
|||
|
|
+ this.scheduledExecutorService.shutdown();
|
|||
|
|
+ return shutdownInner();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private boolean shutdownInner() {
|
|||
|
|
+ return this.rocksDBStorage.shutdown();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException {
|
|||
|
|
+ if (request == null || this.bufferDRList.size() >= BATCH_SIZE) {
|
|||
|
|
+ putMessagePosition();
|
|||
|
|
+ }
|
|||
|
|
+ if (request != null) {
|
|||
|
|
+ this.bufferDRList.add(request);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void putMessagePosition() throws RocksDBException {
|
|||
|
|
+ final int maxRetries = 30;
|
|||
|
|
+ for (int i = 0; i < maxRetries; i++) {
|
|||
|
|
+ if (putMessagePosition0()) {
|
|||
|
|
+ if (this.isCQError) {
|
|||
|
|
+ this.messageStore.getRunningFlags().clearLogicsQueueError();
|
|||
|
|
+ this.isCQError = false;
|
|||
|
|
+ }
|
|||
|
|
+ return;
|
|||
|
|
+ } else {
|
|||
|
|
+ ERROR_LOG.warn("{} put cq Failed. retryTime: {}", i);
|
|||
|
|
+ try {
|
|||
|
|
+ Thread.sleep(100);
|
|||
|
|
+ } catch (InterruptedException ignored) {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ if (!this.isCQError) {
|
|||
|
|
+ ERROR_LOG.error("[BUG] put CQ Failed.");
|
|||
|
|
+ this.messageStore.getRunningFlags().makeLogicsQueueError();
|
|||
|
|
+ this.isCQError = true;
|
|||
|
|
+ }
|
|||
|
|
+ throw new RocksDBException("put CQ Failed");
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private boolean putMessagePosition0() {
|
|||
|
|
+ if (!this.rocksDBStorage.hold()) {
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap = this.tempTopicQueueMaxOffsetMap;
|
|||
|
|
+ try {
|
|||
|
|
+ final List<DispatchRequest> bufferDRList = this.bufferDRList;
|
|||
|
|
+ final int size = bufferDRList.size();
|
|||
|
|
+ if (size == 0) {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+ final List<Pair<ByteBuffer, ByteBuffer>> cqBBPairList = this.cqBBPairList;
|
|||
|
|
+ final List<Pair<ByteBuffer, ByteBuffer>> offsetBBPairList = this.offsetBBPairList;
|
|||
|
|
+ final WriteBatch writeBatch = this.writeBatch;
|
|||
|
|
+
|
|||
|
|
+ long maxPhyOffset = 0;
|
|||
|
|
+ for (int i = size - 1; i >= 0; i--) {
|
|||
|
|
+ final DispatchRequest request = bufferDRList.get(i);
|
|||
|
|
+ final byte[] topicBytes = request.getTopic().getBytes(DataConverter.CHARSET_UTF8);
|
|||
|
|
+
|
|||
|
|
+ this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(cqBBPairList.get(i), topicBytes, request, writeBatch);
|
|||
|
|
+ this.rocksDBConsumeQueueOffsetTable.updateTempTopicQueueMaxOffset(offsetBBPairList.get(i),
|
|||
|
|
+ topicBytes, request, tempTopicQueueMaxOffsetMap);
|
|||
|
|
+
|
|||
|
|
+ final int msgSize = request.getMsgSize();
|
|||
|
|
+ final long phyOffset = request.getCommitLogOffset();
|
|||
|
|
+ if (phyOffset + msgSize >= maxPhyOffset) {
|
|||
|
|
+ maxPhyOffset = phyOffset + msgSize;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset);
|
|||
|
|
+
|
|||
|
|
+ // clear writeBatch in batchPut
|
|||
|
|
+ this.rocksDBStorage.batchPut(writeBatch);
|
|||
|
|
+
|
|||
|
|
+ this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap);
|
|||
|
|
+
|
|||
|
|
+ long storeTimeStamp = bufferDRList.get(size - 1).getStoreTimestamp();
|
|||
|
|
+ if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE
|
|||
|
|
+ || this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) {
|
|||
|
|
+ this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp);
|
|||
|
|
+ }
|
|||
|
|
+ this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp);
|
|||
|
|
+
|
|||
|
|
+ notifyMessageArriveAndClear();
|
|||
|
|
+ return true;
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ ERROR_LOG.error("putMessagePosition0 Failed.", e);
|
|||
|
|
+ return false;
|
|||
|
|
+ } finally {
|
|||
|
|
+ tempTopicQueueMaxOffsetMap.clear();
|
|||
|
|
+ this.rocksDBStorage.release();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void notifyMessageArriveAndClear() {
|
|||
|
|
+ final List<DispatchRequest> bufferDRList = this.bufferDRList;
|
|||
|
|
+ try {
|
|||
|
|
+ for (DispatchRequest dp : bufferDRList) {
|
|||
|
|
+ this.messageStore.notifyMessageArriveIfNecessary(dp);
|
|||
|
|
+ }
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e);
|
|||
|
|
+ } finally {
|
|||
|
|
+ bufferDRList.clear();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException {
|
|||
|
|
+ return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ByteBuffer get(final String topic, final int queueId, final long cqOffset) throws RocksDBException {
|
|||
|
|
+ return this.rocksDBConsumeQueueTable.getCQInKV(topic, queueId, cqOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Ignored, we do not need to recover topicQueueTable and correct minLogicOffset. Because we will correct them
|
|||
|
|
+ * when we use them, we call it lazy correction.
|
|||
|
|
+ * @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short)
|
|||
|
|
+ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int)
|
|||
|
|
+ */
|
|||
|
|
+ @Override
|
|||
|
|
+ public void recoverOffsetTable(long minPhyOffset) {
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void destroy() {
|
|||
|
|
+ try {
|
|||
|
|
+ shutdownInner();
|
|||
|
|
+ FileUtils.deleteDirectory(new File(this.storePath));
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ ERROR_LOG.error("destroy cq Failed. {}", this.storePath, e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException {
|
|||
|
|
+ String topic = consumeQueue.getTopic();
|
|||
|
|
+ int queueId = consumeQueue.getQueueId();
|
|||
|
|
+ if (StringUtils.isEmpty(topic) || queueId < 0 || !this.rocksDBStorage.hold()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ WriteBatch writeBatch = new WriteBatch();
|
|||
|
|
+ try {
|
|||
|
|
+ this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch);
|
|||
|
|
+ this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch);
|
|||
|
|
+
|
|||
|
|
+ this.rocksDBStorage.batchPut(writeBatch);
|
|||
|
|
+ } catch (RocksDBException e) {
|
|||
|
|
+ ERROR_LOG.error("kv deleteTopic {} Failed.", topic, e);
|
|||
|
|
+ throw e;
|
|||
|
|
+ } finally {
|
|||
|
|
+ writeBatch.close();
|
|||
|
|
+ this.rocksDBStorage.release();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) {
|
|||
|
|
+ try {
|
|||
|
|
+ this.rocksDBStorage.flushWAL();
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ }
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void checkSelf() {
|
|||
|
|
+ // ignored
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) {
|
|||
|
|
+ // ignored
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * We do not need to truncate dirty CQ in RocksDBConsumeQueueTable, Because dirty CQ in RocksDBConsumeQueueTable
|
|||
|
|
+ * will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted.
|
|||
|
|
+ * But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in
|
|||
|
|
+ * RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset).
|
|||
|
|
+ * @param offsetToTruncate
|
|||
|
|
+ * @throws RocksDBException
|
|||
|
|
+ */
|
|||
|
|
+ @Override
|
|||
|
|
+ public void truncateDirty(long offsetToTruncate) throws RocksDBException {
|
|||
|
|
+ long maxPhyOffsetInRocksdb = getMaxPhyOffsetInConsumeQueue();
|
|||
|
|
+ if (offsetToTruncate >= maxPhyOffsetInRocksdb) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ this.rocksDBConsumeQueueOffsetTable.truncateDirty(offsetToTruncate);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void cleanExpired(final long minPhyOffset) {
|
|||
|
|
+ this.rocksDBStorage.manualCompaction(minPhyOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException {
|
|||
|
|
+ final long minPhysicOffset = this.messageStore.getMinPhyOffset();
|
|||
|
|
+ long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId);
|
|||
|
|
+ Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId);
|
|||
|
|
+ if (high == null || high == -1) {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+ return this.rocksDBConsumeQueueTable.binarySearchInCQByTime(topic, queueId, high, low, timestamp, minPhysicOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException {
|
|||
|
|
+ Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId);
|
|||
|
|
+ return (maxOffset != null) ? maxOffset + 1 : 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMinOffsetInQueue(String topic, int queueId) throws RocksDBException {
|
|||
|
|
+ return this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) {
|
|||
|
|
+ return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(topic, queueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getMaxPhyOffsetInConsumeQueue() throws RocksDBException {
|
|||
|
|
+ return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) {
|
|||
|
|
+ ConcurrentMap<Integer, ConsumeQueueInterface> map = this.consumeQueueTable.get(topic);
|
|||
|
|
+ if (null == map) {
|
|||
|
|
+ ConcurrentMap<Integer, ConsumeQueueInterface> newMap = new ConcurrentHashMap<>(128);
|
|||
|
|
+ ConcurrentMap<Integer, ConsumeQueueInterface> oldMap = this.consumeQueueTable.putIfAbsent(topic, newMap);
|
|||
|
|
+ if (oldMap != null) {
|
|||
|
|
+ map = oldMap;
|
|||
|
|
+ } else {
|
|||
|
|
+ map = newMap;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ ConsumeQueueInterface logic = map.get(queueId);
|
|||
|
|
+ if (logic != null) {
|
|||
|
|
+ return logic;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ ConsumeQueueInterface newLogic = new RocksDBConsumeQueue(this.messageStore, topic, queueId);
|
|||
|
|
+ ConsumeQueueInterface oldLogic = map.putIfAbsent(queueId, newLogic);
|
|||
|
|
+
|
|||
|
|
+ return oldLogic != null ? oldLogic : newLogic;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long rollNextFile(ConsumeQueueInterface consumeQueue, long offset) {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) {
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public long getTotalSize() {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..0a735ea27
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java
|
|||
|
|
@@ -0,0 +1,312 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.queue;
|
|||
|
|
+
|
|||
|
|
+import java.nio.ByteBuffer;
|
|||
|
|
+import java.util.ArrayList;
|
|||
|
|
+import java.util.List;
|
|||
|
|
+
|
|||
|
|
+import org.apache.rocketmq.common.Pair;
|
|||
|
|
+import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
+import org.apache.rocketmq.store.DispatchRequest;
|
|||
|
|
+import org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset;
|
|||
|
|
+import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
|
|||
|
|
+import org.rocksdb.ColumnFamilyHandle;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+import org.rocksdb.WriteBatch;
|
|||
|
|
+
|
|||
|
|
+import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8;
|
|||
|
|
+import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_0;
|
|||
|
|
+import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1;
|
|||
|
|
+import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_2;
|
|||
|
|
+
|
|||
|
|
+/**
|
|||
|
|
+ * We use RocksDBConsumeQueueTable to store cqUnit.
|
|||
|
|
+ */
|
|||
|
|
+public class RocksDBConsumeQueueTable {
|
|||
|
|
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
|
|||
|
|
+ private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
|
|||
|
|
+ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Rocksdb ConsumeQueue's store unit. Format:
|
|||
|
|
+ *
|
|||
|
|
+ * <pre>
|
|||
|
|
+ * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬───────────────────────┐
|
|||
|
|
+ * │ Topic Bytes Array Size │ CTRL_1 │ Topic Bytes Array │ CTRL_1 │ QueueId │ CTRL_1 │ ConsumeQueue Offset │
|
|||
|
|
+ * │ (4 Bytes) │ (1 Bytes) │ (n Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ (8 Bytes) │
|
|||
|
|
+ * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴───────────────────────┤
|
|||
|
|
+ * │ Key Unit │
|
|||
|
|
+ * │ │
|
|||
|
|
+ * </pre>
|
|||
|
|
+ *
|
|||
|
|
+ * <pre>
|
|||
|
|
+ * ┌─────────────────────────────┬───────────────────┬──────────────────┬──────────────────┐
|
|||
|
|
+ * │ CommitLog Physical Offset │ Body Size │ Tag HashCode │ Msg Store Time │
|
|||
|
|
+ * │ (8 Bytes) │ (4 Bytes) │ (8 Bytes) │ (8 Bytes) │
|
|||
|
|
+ * ├─────────────────────────────┴───────────────────┴──────────────────┴──────────────────┤
|
|||
|
|
+ * │ Value Unit │
|
|||
|
|
+ * │ │
|
|||
|
|
+ * </pre>
|
|||
|
|
+ * ConsumeQueue's store unit. Size:
|
|||
|
|
+ * CommitLog Physical Offset(8) + Body Size(4) + Tag HashCode(8) + Msg Store Time(8) = 28 Bytes
|
|||
|
|
+ */
|
|||
|
|
+ private static final int PHY_OFFSET_OFFSET = 0;
|
|||
|
|
+ private static final int PHY_MSG_LEN_OFFSET = 8;
|
|||
|
|
+ private static final int MSG_TAG_HASHCODE_OFFSET = 12;
|
|||
|
|
+ private static final int MSG_STORE_TIME_SIZE_OFFSET = 20;
|
|||
|
|
+ public static final int CQ_UNIT_SIZE = 8 + 4 + 8 + 8;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬───────────────────────┐
|
|||
|
|
+ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_1 │ ConsumeQueue Offset │
|
|||
|
|
+ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ (8 Bytes) │
|
|||
|
|
+ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴───────────────────────┤
|
|||
|
|
+ */
|
|||
|
|
+ private static final int CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1 + 8;
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────────────┐
|
|||
|
|
+ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_0(CTRL_2) │
|
|||
|
|
+ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │
|
|||
|
|
+ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────────────┤
|
|||
|
|
+ */
|
|||
|
|
+ private static final int DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1;
|
|||
|
|
+
|
|||
|
|
+ private final ConsumeQueueRocksDBStorage rocksDBStorage;
|
|||
|
|
+ private final DefaultMessageStore messageStore;
|
|||
|
|
+
|
|||
|
|
+ private ColumnFamilyHandle defaultCFH;
|
|||
|
|
+
|
|||
|
|
+ public RocksDBConsumeQueueTable(ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) {
|
|||
|
|
+ this.rocksDBStorage = rocksDBStorage;
|
|||
|
|
+ this.messageStore = messageStore;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void load() {
|
|||
|
|
+ this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void buildAndPutCQByteBuffer(final Pair<ByteBuffer, ByteBuffer> cqBBPair,
|
|||
|
|
+ final byte[] topicBytes, final DispatchRequest request, final WriteBatch writeBatch) throws RocksDBException {
|
|||
|
|
+ final ByteBuffer cqKey = cqBBPair.getObject1();
|
|||
|
|
+ buildCQKeyByteBuffer(cqKey, topicBytes, request.getQueueId(), request.getConsumeQueueOffset());
|
|||
|
|
+
|
|||
|
|
+ final ByteBuffer cqValue = cqBBPair.getObject2();
|
|||
|
|
+ buildCQValueByteBuffer(cqValue, request.getCommitLogOffset(), request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp());
|
|||
|
|
+
|
|||
|
|
+ writeBatch.put(this.defaultCFH, cqKey, cqValue);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException {
|
|||
|
|
+ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
|
|||
|
|
+ final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset);
|
|||
|
|
+ byte[] value = this.rocksDBStorage.getCQ(keyBB.array());
|
|||
|
|
+ return (value != null) ? ByteBuffer.wrap(value) : null;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException {
|
|||
|
|
+ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
|
|||
|
|
+ final List<ColumnFamilyHandle> defaultCFHList = new ArrayList(num);
|
|||
|
|
+ final ByteBuffer[] resultList = new ByteBuffer[num];
|
|||
|
|
+ final List<Integer> kvIndexList = new ArrayList(num);
|
|||
|
|
+ final List<byte[]> kvKeyList = new ArrayList(num);
|
|||
|
|
+ for (int i = 0; i < num; i++) {
|
|||
|
|
+ final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i);
|
|||
|
|
+ kvIndexList.add(i);
|
|||
|
|
+ kvKeyList.add(keyBB.array());
|
|||
|
|
+ defaultCFHList.add(this.defaultCFH);
|
|||
|
|
+ }
|
|||
|
|
+ int keyNum = kvIndexList.size();
|
|||
|
|
+ if (keyNum > 0) {
|
|||
|
|
+ List<byte[]> kvValueList = this.rocksDBStorage.multiGet(defaultCFHList, kvKeyList);
|
|||
|
|
+ final int valueNum = kvValueList.size();
|
|||
|
|
+ if (keyNum != valueNum) {
|
|||
|
|
+ throw new RocksDBException("rocksdb bug, multiGet");
|
|||
|
|
+ }
|
|||
|
|
+ for (int i = 0; i < valueNum; i++) {
|
|||
|
|
+ byte[] value = kvValueList.get(i);
|
|||
|
|
+ if (value == null) {
|
|||
|
|
+ continue;
|
|||
|
|
+ }
|
|||
|
|
+ ByteBuffer byteBuffer = ByteBuffer.wrap(value);
|
|||
|
|
+ resultList[kvIndexList.get(i)] = byteBuffer;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ final int resultSize = resultList.length;
|
|||
|
|
+ List<ByteBuffer> bbValueList = new ArrayList(resultSize);
|
|||
|
|
+ for (int i = 0; i < resultSize; i++) {
|
|||
|
|
+ ByteBuffer byteBuffer = resultList[i];
|
|||
|
|
+ if (byteBuffer == null) {
|
|||
|
|
+ break;
|
|||
|
|
+ }
|
|||
|
|
+ bbValueList.add(byteBuffer);
|
|||
|
|
+ }
|
|||
|
|
+ return bbValueList;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * When topic is deleted, we clean up its CqUnit in rocksdb.
|
|||
|
|
+ * @param topic
|
|||
|
|
+ * @param queueId
|
|||
|
|
+ * @throws RocksDBException
|
|||
|
|
+ */
|
|||
|
|
+ public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException {
|
|||
|
|
+ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
|
|||
|
|
+ final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId);
|
|||
|
|
+ final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId);
|
|||
|
|
+
|
|||
|
|
+ writeBatch.deleteRange(this.defaultCFH, cqStartKey.array(), cqEndKey.array());
|
|||
|
|
+
|
|||
|
|
+ log.info("Rocksdb consumeQueue table delete topic. {}, {}", topic, queueId);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public long binarySearchInCQByTime(String topic, int queueId, long high, long low, long timestamp,
|
|||
|
|
+ long minPhysicOffset) throws RocksDBException {
|
|||
|
|
+ long result = 0;
|
|||
|
|
+ long targetOffset = -1L, leftOffset = -1L, rightOffset = -1L;
|
|||
|
|
+ long leftValue = -1L, rightValue = -1L;
|
|||
|
|
+ while (high >= low) {
|
|||
|
|
+ long midOffset = low + ((high - low) >>> 1);
|
|||
|
|
+ ByteBuffer byteBuffer = getCQInKV(topic, queueId, midOffset);
|
|||
|
|
+ if (byteBuffer == null) {
|
|||
|
|
+ ERROR_LOG.warn("binarySearchInCQByTimeStamp Failed. topic: {}, queueId: {}, timestamp: {}, result: null",
|
|||
|
|
+ topic, queueId, timestamp);
|
|||
|
|
+ low = midOffset + 1;
|
|||
|
|
+ continue;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET);
|
|||
|
|
+ if (phyOffset < minPhysicOffset) {
|
|||
|
|
+ low = midOffset + 1;
|
|||
|
|
+ leftOffset = midOffset;
|
|||
|
|
+ continue;
|
|||
|
|
+ }
|
|||
|
|
+ long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);
|
|||
|
|
+ if (storeTime < 0) {
|
|||
|
|
+ return 0;
|
|||
|
|
+ } else if (storeTime == timestamp) {
|
|||
|
|
+ targetOffset = midOffset;
|
|||
|
|
+ break;
|
|||
|
|
+ } else if (storeTime > timestamp) {
|
|||
|
|
+ high = midOffset - 1;
|
|||
|
|
+ rightOffset = midOffset;
|
|||
|
|
+ rightValue = storeTime;
|
|||
|
|
+ } else {
|
|||
|
|
+ low = midOffset + 1;
|
|||
|
|
+ leftOffset = midOffset;
|
|||
|
|
+ leftValue = storeTime;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ if (targetOffset != -1) {
|
|||
|
|
+ result = targetOffset;
|
|||
|
|
+ } else {
|
|||
|
|
+ if (leftValue == -1) {
|
|||
|
|
+ result = rightOffset;
|
|||
|
|
+ } else if (rightValue == -1) {
|
|||
|
|
+ result = leftOffset;
|
|||
|
|
+ } else {
|
|||
|
|
+ result = Math.abs(timestamp - leftValue) > Math.abs(timestamp - rightValue) ? rightOffset : leftOffset;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ return result;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public PhyAndCQOffset binarySearchInCQ(String topic, int queueId, long high, long low, long targetPhyOffset,
|
|||
|
|
+ boolean min) throws RocksDBException {
|
|||
|
|
+ long resultCQOffset = -1L;
|
|||
|
|
+ long resultPhyOffset = -1L;
|
|||
|
|
+ while (high >= low) {
|
|||
|
|
+ long midCQOffset = low + ((high - low) >>> 1);
|
|||
|
|
+ ByteBuffer byteBuffer = getCQInKV(topic, queueId, midCQOffset);
|
|||
|
|
+ if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
|
|||
|
|
+ ROCKSDB_LOG.warn("binarySearchInCQ. {}, {}, {}, {}, {}", topic, queueId, midCQOffset, low, high);
|
|||
|
|
+ }
|
|||
|
|
+ if (byteBuffer == null) {
|
|||
|
|
+ low = midCQOffset + 1;
|
|||
|
|
+ continue;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ final long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET);
|
|||
|
|
+ if (phyOffset == targetPhyOffset) {
|
|||
|
|
+ if (min) {
|
|||
|
|
+ resultCQOffset = midCQOffset;
|
|||
|
|
+ resultPhyOffset = phyOffset;
|
|||
|
|
+ }
|
|||
|
|
+ break;
|
|||
|
|
+ } else if (phyOffset > targetPhyOffset) {
|
|||
|
|
+ high = midCQOffset - 1;
|
|||
|
|
+ if (min) {
|
|||
|
|
+ resultCQOffset = midCQOffset;
|
|||
|
|
+ resultPhyOffset = phyOffset;
|
|||
|
|
+ }
|
|||
|
|
+ } else {
|
|||
|
|
+ low = midCQOffset + 1;
|
|||
|
|
+ if (!min) {
|
|||
|
|
+ resultCQOffset = midCQOffset;
|
|||
|
|
+ resultPhyOffset = phyOffset;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ return new PhyAndCQOffset(resultPhyOffset, resultCQOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static Pair<ByteBuffer, ByteBuffer> getCQByteBufferPair() {
|
|||
|
|
+ ByteBuffer cqKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN);
|
|||
|
|
+ ByteBuffer cqValue = ByteBuffer.allocateDirect(CQ_UNIT_SIZE);
|
|||
|
|
+ return new Pair<>(cqKey, cqValue);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ByteBuffer buildCQKeyByteBuffer(final byte[] topicBytes, final int queueId, final long cqOffset) {
|
|||
|
|
+ final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);
|
|||
|
|
+ buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset);
|
|||
|
|
+ return byteBuffer;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildCQKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) {
|
|||
|
|
+ byteBuffer.position(0).limit(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);
|
|||
|
|
+ buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildCQKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) {
|
|||
|
|
+ byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(CTRL_1).putLong(cqOffset);
|
|||
|
|
+ byteBuffer.flip();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildCQValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, final long tagsCode, final long storeTimestamp) {
|
|||
|
|
+ byteBuffer.position(0).limit(CQ_UNIT_SIZE);
|
|||
|
|
+ buildCQValueByteBuffer0(byteBuffer, phyOffset, msgSize, tagsCode, storeTimestamp);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void buildCQValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize,
|
|||
|
|
+ final long tagsCode, final long storeTimestamp) {
|
|||
|
|
+ byteBuffer.putLong(phyOffset).putInt(msgSize).putLong(tagsCode).putLong(storeTimestamp);
|
|||
|
|
+ byteBuffer.flip();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private ByteBuffer buildDeleteCQKey(final boolean start, final byte[] topicBytes, final int queueId) {
|
|||
|
|
+ final ByteBuffer byteBuffer = ByteBuffer.allocate(DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);
|
|||
|
|
+
|
|||
|
|
+ byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(start ? CTRL_0 : CTRL_2);
|
|||
|
|
+ byteBuffer.flip();
|
|||
|
|
+ return byteBuffer;
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..aa796c4d3
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java
|
|||
|
|
@@ -0,0 +1,47 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.rocksdb;
|
|||
|
|
+
|
|||
|
|
+import org.apache.rocketmq.common.constant.LoggerName;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
+import org.apache.rocketmq.store.MessageStore;
|
|||
|
|
+import org.rocksdb.AbstractCompactionFilter;
|
|||
|
|
+import org.rocksdb.AbstractCompactionFilterFactory;
|
|||
|
|
+import org.rocksdb.RemoveConsumeQueueCompactionFilter;
|
|||
|
|
+
|
|||
|
|
+public class ConsumeQueueCompactionFilterFactory extends AbstractCompactionFilterFactory<RemoveConsumeQueueCompactionFilter> {
|
|||
|
|
+ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
|
|||
|
|
+ private final MessageStore messageStore;
|
|||
|
|
+
|
|||
|
|
+ public ConsumeQueueCompactionFilterFactory(final MessageStore messageStore) {
|
|||
|
|
+ this.messageStore = messageStore;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String name() {
|
|||
|
|
+ return "ConsumeQueueCompactionFilterFactory";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public RemoveConsumeQueueCompactionFilter createCompactionFilter(final AbstractCompactionFilter.Context context) {
|
|||
|
|
+ long minPhyOffset = this.messageStore.getMinPhyOffset();
|
|||
|
|
+ LOGGER.info("manualCompaction minPhyOffset: {}, isFull: {}, isManual: {}",
|
|||
|
|
+ minPhyOffset, context.isFullCompaction(), context.isManualCompaction());
|
|||
|
|
+ return new RemoveConsumeQueueCompactionFilter(minPhyOffset);
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..362684560
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java
|
|||
|
|
@@ -0,0 +1,133 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.rocksdb;
|
|||
|
|
+
|
|||
|
|
+import java.util.ArrayList;
|
|||
|
|
+import java.util.List;
|
|||
|
|
+
|
|||
|
|
+import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
|
|||
|
|
+import org.apache.rocketmq.common.utils.DataConverter;
|
|||
|
|
+import org.apache.rocketmq.store.MessageStore;
|
|||
|
|
+import org.rocksdb.ColumnFamilyDescriptor;
|
|||
|
|
+import org.rocksdb.ColumnFamilyHandle;
|
|||
|
|
+import org.rocksdb.ColumnFamilyOptions;
|
|||
|
|
+import org.rocksdb.CompactRangeOptions;
|
|||
|
|
+import org.rocksdb.ReadOptions;
|
|||
|
|
+import org.rocksdb.RocksDB;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
+import org.rocksdb.RocksIterator;
|
|||
|
|
+import org.rocksdb.WriteBatch;
|
|||
|
|
+import org.rocksdb.WriteOptions;
|
|||
|
|
+
|
|||
|
|
+public class ConsumeQueueRocksDBStorage extends AbstractRocksDBStorage {
|
|||
|
|
+ private final MessageStore messageStore;
|
|||
|
|
+ private volatile ColumnFamilyHandle offsetCFHandle;
|
|||
|
|
+
|
|||
|
|
+ public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath, final int prefixLen) {
|
|||
|
|
+ this.messageStore = messageStore;
|
|||
|
|
+ this.dbPath = dbPath;
|
|||
|
|
+ this.readOnly = false;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void initOptions() {
|
|||
|
|
+ this.options = RocksDBOptionsFactory.createDBOptions();
|
|||
|
|
+
|
|||
|
|
+ this.writeOptions = new WriteOptions();
|
|||
|
|
+ this.writeOptions.setSync(false);
|
|||
|
|
+ this.writeOptions.setDisableWAL(true);
|
|||
|
|
+ this.writeOptions.setNoSlowdown(true);
|
|||
|
|
+
|
|||
|
|
+ this.totalOrderReadOptions = new ReadOptions();
|
|||
|
|
+ this.totalOrderReadOptions.setPrefixSameAsStart(false);
|
|||
|
|
+ this.totalOrderReadOptions.setTotalOrderSeek(false);
|
|||
|
|
+
|
|||
|
|
+ this.compactRangeOptions = new CompactRangeOptions();
|
|||
|
|
+ this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce);
|
|||
|
|
+ this.compactRangeOptions.setAllowWriteStall(true);
|
|||
|
|
+ this.compactRangeOptions.setExclusiveManualCompaction(false);
|
|||
|
|
+ this.compactRangeOptions.setChangeLevel(true);
|
|||
|
|
+ this.compactRangeOptions.setTargetLevel(-1);
|
|||
|
|
+ this.compactRangeOptions.setMaxSubcompactions(4);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ protected boolean postLoad() {
|
|||
|
|
+ try {
|
|||
|
|
+ UtilAll.ensureDirOK(this.dbPath);
|
|||
|
|
+
|
|||
|
|
+ initOptions();
|
|||
|
|
+
|
|||
|
|
+ final List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList();
|
|||
|
|
+
|
|||
|
|
+ ColumnFamilyOptions cqCfOptions = RocksDBOptionsFactory.createCQCFOptions(this.messageStore);
|
|||
|
|
+ this.cfOptions.add(cqCfOptions);
|
|||
|
|
+ cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cqCfOptions));
|
|||
|
|
+
|
|||
|
|
+ ColumnFamilyOptions offsetCfOptions = RocksDBOptionsFactory.createOffsetCFOptions();
|
|||
|
|
+ this.cfOptions.add(offsetCfOptions);
|
|||
|
|
+ cfDescriptors.add(new ColumnFamilyDescriptor("offset".getBytes(DataConverter.CHARSET_UTF8), offsetCfOptions));
|
|||
|
|
+
|
|||
|
|
+ final List<ColumnFamilyHandle> cfHandles = new ArrayList();
|
|||
|
|
+ open(cfDescriptors, cfHandles);
|
|||
|
|
+
|
|||
|
|
+ this.defaultCFHandle = cfHandles.get(0);
|
|||
|
|
+ this.offsetCFHandle = cfHandles.get(1);
|
|||
|
|
+ } catch (final Exception e) {
|
|||
|
|
+ LOGGER.error("postLoad Failed. {}", this.dbPath, e);
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+ return true;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ protected void preShutdown() {
|
|||
|
|
+ this.offsetCFHandle.close();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public byte[] getCQ(final byte[] keyBytes) throws RocksDBException {
|
|||
|
|
+ return get(this.defaultCFHandle, this.totalOrderReadOptions, keyBytes);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public byte[] getOffset(final byte[] keyBytes) throws RocksDBException {
|
|||
|
|
+ return get(this.offsetCFHandle, this.totalOrderReadOptions, keyBytes);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public List<byte[]> multiGet(final List<ColumnFamilyHandle> cfhList, final List<byte[]> keys) throws RocksDBException {
|
|||
|
|
+ return multiGet(this.totalOrderReadOptions, cfhList, keys);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void batchPut(final WriteBatch batch) throws RocksDBException {
|
|||
|
|
+ batchPut(this.writeOptions, batch);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public void manualCompaction(final long minPhyOffset) {
|
|||
|
|
+ try {
|
|||
|
|
+ manualCompaction(minPhyOffset, this.compactRangeOptions);
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ LOGGER.error("manualCompaction Failed. minPhyOffset: {}", minPhyOffset, e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public RocksIterator seekOffsetCF() {
|
|||
|
|
+ return this.db.newIterator(this.offsetCFHandle, this.totalOrderReadOptions);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public ColumnFamilyHandle getOffsetCFHandle() {
|
|||
|
|
+ return this.offsetCFHandle;
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
\ No newline at end of file
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..a3a99d334
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java
|
|||
|
|
@@ -0,0 +1,161 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.store.rocksdb;
|
|||
|
|
+
|
|||
|
|
+import org.apache.rocketmq.common.config.ConfigRocksDBStorage;
|
|||
|
|
+import org.apache.rocketmq.store.MessageStore;
|
|||
|
|
+import org.rocksdb.BlockBasedTableConfig;
|
|||
|
|
+import org.rocksdb.BloomFilter;
|
|||
|
|
+import org.rocksdb.ColumnFamilyOptions;
|
|||
|
|
+import org.rocksdb.CompactionOptionsUniversal;
|
|||
|
|
+import org.rocksdb.CompactionStopStyle;
|
|||
|
|
+import org.rocksdb.CompactionStyle;
|
|||
|
|
+import org.rocksdb.CompressionType;
|
|||
|
|
+import org.rocksdb.DBOptions;
|
|||
|
|
+import org.rocksdb.DataBlockIndexType;
|
|||
|
|
+import org.rocksdb.IndexType;
|
|||
|
|
+import org.rocksdb.InfoLogLevel;
|
|||
|
|
+import org.rocksdb.LRUCache;
|
|||
|
|
+import org.rocksdb.RateLimiter;
|
|||
|
|
+import org.rocksdb.SkipListMemTableConfig;
|
|||
|
|
+import org.rocksdb.Statistics;
|
|||
|
|
+import org.rocksdb.StatsLevel;
|
|||
|
|
+import org.rocksdb.StringAppendOperator;
|
|||
|
|
+import org.rocksdb.WALRecoveryMode;
|
|||
|
|
+import org.rocksdb.util.SizeUnit;
|
|||
|
|
+
|
|||
|
|
+public class RocksDBOptionsFactory {
|
|||
|
|
+
|
|||
|
|
+ public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageStore) {
|
|||
|
|
+ BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig().
|
|||
|
|
+ setFormatVersion(5).
|
|||
|
|
+ setIndexType(IndexType.kBinarySearch).
|
|||
|
|
+ setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash).
|
|||
|
|
+ setDataBlockHashTableUtilRatio(0.75).
|
|||
|
|
+ setBlockSize(32 * SizeUnit.KB).
|
|||
|
|
+ setMetadataBlockSize(4 * SizeUnit.KB).
|
|||
|
|
+ setFilterPolicy(new BloomFilter(16, false)).
|
|||
|
|
+ setCacheIndexAndFilterBlocks(false).
|
|||
|
|
+ setCacheIndexAndFilterBlocksWithHighPriority(true).
|
|||
|
|
+ setPinL0FilterAndIndexBlocksInCache(false).
|
|||
|
|
+ setPinTopLevelIndexAndFilter(true).
|
|||
|
|
+ setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false)).
|
|||
|
|
+ setWholeKeyFiltering(true);
|
|||
|
|
+
|
|||
|
|
+ ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
|
|||
|
|
+ CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal();
|
|||
|
|
+ compactionOption.setSizeRatio(100).
|
|||
|
|
+ setMaxSizeAmplificationPercent(25).
|
|||
|
|
+ setAllowTrivialMove(true).
|
|||
|
|
+ setMinMergeWidth(2).
|
|||
|
|
+ setMaxMergeWidth(Integer.MAX_VALUE).
|
|||
|
|
+ setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize).
|
|||
|
|
+ setCompressionSizePercent(-1);
|
|||
|
|
+ return columnFamilyOptions.setMaxWriteBufferNumber(4).
|
|||
|
|
+ setWriteBufferSize(128 * SizeUnit.MB).
|
|||
|
|
+ setMinWriteBufferNumberToMerge(1).
|
|||
|
|
+ setTableFormatConfig(blockBasedTableConfig).
|
|||
|
|
+ setMemTableConfig(new SkipListMemTableConfig()).
|
|||
|
|
+ setCompressionType(CompressionType.LZ4_COMPRESSION).
|
|||
|
|
+ setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION).
|
|||
|
|
+ setNumLevels(7).
|
|||
|
|
+ setCompactionStyle(CompactionStyle.UNIVERSAL).
|
|||
|
|
+ setCompactionOptionsUniversal(compactionOption).
|
|||
|
|
+ setMaxCompactionBytes(100 * SizeUnit.GB).
|
|||
|
|
+ setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB).
|
|||
|
|
+ setHardPendingCompactionBytesLimit(256 * SizeUnit.GB).
|
|||
|
|
+ setLevel0FileNumCompactionTrigger(2).
|
|||
|
|
+ setLevel0SlowdownWritesTrigger(8).
|
|||
|
|
+ setLevel0StopWritesTrigger(10).
|
|||
|
|
+ setTargetFileSizeBase(256 * SizeUnit.MB).
|
|||
|
|
+ setTargetFileSizeMultiplier(2).
|
|||
|
|
+ setMergeOperator(new StringAppendOperator()).
|
|||
|
|
+ setCompactionFilterFactory(new ConsumeQueueCompactionFilterFactory(messageStore)).
|
|||
|
|
+ setReportBgIoStats(true).
|
|||
|
|
+ setOptimizeFiltersForHits(true);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public static ColumnFamilyOptions createOffsetCFOptions() {
|
|||
|
|
+ BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig().
|
|||
|
|
+ setFormatVersion(5).
|
|||
|
|
+ setIndexType(IndexType.kBinarySearch).
|
|||
|
|
+ setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch).
|
|||
|
|
+ setBlockSize(32 * SizeUnit.KB).
|
|||
|
|
+ setFilterPolicy(new BloomFilter(16, false)).
|
|||
|
|
+ setCacheIndexAndFilterBlocks(false).
|
|||
|
|
+ setCacheIndexAndFilterBlocksWithHighPriority(true).
|
|||
|
|
+ setPinL0FilterAndIndexBlocksInCache(false).
|
|||
|
|
+ setPinTopLevelIndexAndFilter(true).
|
|||
|
|
+ setBlockCache(new LRUCache(128 * SizeUnit.MB, 8, false)).
|
|||
|
|
+ setWholeKeyFiltering(true);
|
|||
|
|
+
|
|||
|
|
+ ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
|
|||
|
|
+ return columnFamilyOptions.setMaxWriteBufferNumber(4).
|
|||
|
|
+ setWriteBufferSize(64 * SizeUnit.MB).
|
|||
|
|
+ setMinWriteBufferNumberToMerge(1).
|
|||
|
|
+ setTableFormatConfig(blockBasedTableConfig).
|
|||
|
|
+ setMemTableConfig(new SkipListMemTableConfig()).
|
|||
|
|
+ setCompressionType(CompressionType.NO_COMPRESSION).
|
|||
|
|
+ setNumLevels(7).
|
|||
|
|
+ setCompactionStyle(CompactionStyle.LEVEL).
|
|||
|
|
+ setLevel0FileNumCompactionTrigger(2).
|
|||
|
|
+ setLevel0SlowdownWritesTrigger(8).
|
|||
|
|
+ setLevel0StopWritesTrigger(10).
|
|||
|
|
+ setTargetFileSizeBase(64 * SizeUnit.MB).
|
|||
|
|
+ setTargetFileSizeMultiplier(2).
|
|||
|
|
+ setMaxBytesForLevelBase(256 * SizeUnit.MB).
|
|||
|
|
+ setMaxBytesForLevelMultiplier(2).
|
|||
|
|
+ setMergeOperator(new StringAppendOperator()).
|
|||
|
|
+ setInplaceUpdateSupport(true);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ /**
|
|||
|
|
+ * Create a rocksdb db options, the user must take care to close it after closing db.
|
|||
|
|
+ * @return
|
|||
|
|
+ */
|
|||
|
|
+ public static DBOptions createDBOptions() {
|
|||
|
|
+ //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
|
|||
|
|
+ // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java
|
|||
|
|
+ DBOptions options = new DBOptions();
|
|||
|
|
+ Statistics statistics = new Statistics();
|
|||
|
|
+ statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
|
|||
|
|
+ return options.
|
|||
|
|
+ setDbLogDir(ConfigRocksDBStorage.getDBLogDir()).
|
|||
|
|
+ setInfoLogLevel(InfoLogLevel.INFO_LEVEL).
|
|||
|
|
+ setWalRecoveryMode(WALRecoveryMode.PointInTimeRecovery).
|
|||
|
|
+ setManualWalFlush(true).
|
|||
|
|
+ setMaxTotalWalSize(0).
|
|||
|
|
+ setWalSizeLimitMB(0).
|
|||
|
|
+ setWalTtlSeconds(0).
|
|||
|
|
+ setCreateIfMissing(true).
|
|||
|
|
+ setCreateMissingColumnFamilies(true).
|
|||
|
|
+ setMaxOpenFiles(-1).
|
|||
|
|
+ setMaxLogFileSize(1 * SizeUnit.GB).
|
|||
|
|
+ setKeepLogFileNum(5).
|
|||
|
|
+ setMaxManifestFileSize(1 * SizeUnit.GB).
|
|||
|
|
+ setAllowConcurrentMemtableWrite(false).
|
|||
|
|
+ setStatistics(statistics).
|
|||
|
|
+ setAtomicFlush(true).
|
|||
|
|
+ setMaxBackgroundJobs(32).
|
|||
|
|
+ setMaxSubcompactions(8).
|
|||
|
|
+ setParanoidChecks(true).
|
|||
|
|
+ setDelayedWriteRate(16 * SizeUnit.MB).
|
|||
|
|
+ setRateLimiter(new RateLimiter(100 * SizeUnit.MB)).
|
|||
|
|
+ setUseDirectIoForFlushAndCompaction(false).
|
|||
|
|
+ setUseDirectReads(false);
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
index ac4c61cd6..3ab51a26d 100644
|
|||
|
|
--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
+++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
|
|||
|
|
@@ -57,7 +57,6 @@ import org.apache.rocketmq.common.topic.TopicValidator;
|
|||
|
|
import org.apache.rocketmq.common.utils.ThreadUtils;
|
|||
|
|
import org.apache.rocketmq.logging.org.slf4j.Logger;
|
|||
|
|
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
|
|||
|
|
-import org.apache.rocketmq.store.ConsumeQueue;
|
|||
|
|
import org.apache.rocketmq.store.DefaultMessageStore;
|
|||
|
|
import org.apache.rocketmq.store.MessageStore;
|
|||
|
|
import org.apache.rocketmq.store.PutMessageResult;
|
|||
|
|
@@ -66,6 +65,9 @@ import org.apache.rocketmq.store.config.BrokerRole;
|
|||
|
|
import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
import org.apache.rocketmq.store.logfile.MappedFile;
|
|||
|
|
import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager;
|
|||
|
|
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
|
|||
|
|
+import org.apache.rocketmq.store.queue.CqUnit;
|
|||
|
|
+import org.apache.rocketmq.store.queue.ReferredIterator;
|
|||
|
|
import org.apache.rocketmq.store.stats.BrokerStatsManager;
|
|||
|
|
import org.apache.rocketmq.store.util.PerfCounter;
|
|||
|
|
|
|||
|
|
@@ -333,7 +335,7 @@ public class TimerMessageStore {
|
|||
|
|
// if not, use cq offset.
|
|||
|
|
long msgQueueOffset = messageExt.getQueueOffset();
|
|||
|
|
int queueId = messageExt.getQueueId();
|
|||
|
|
- ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId);
|
|||
|
|
+ ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId);
|
|||
|
|
if (null == cq) {
|
|||
|
|
return msgQueueOffset;
|
|||
|
|
}
|
|||
|
|
@@ -346,15 +348,18 @@ public class TimerMessageStore {
|
|||
|
|
offsetPy, sizePy);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
- SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(tmpOffset);
|
|||
|
|
- if (null == bufferCQ) {
|
|||
|
|
- // offset in msg may be greater than offset of cq.
|
|||
|
|
- tmpOffset -= 1;
|
|||
|
|
- continue;
|
|||
|
|
- }
|
|||
|
|
+ ReferredIterator<CqUnit> iterator = null;
|
|||
|
|
try {
|
|||
|
|
- long offsetPyTemp = bufferCQ.getByteBuffer().getLong();
|
|||
|
|
- int sizePyTemp = bufferCQ.getByteBuffer().getInt();
|
|||
|
|
+ iterator = cq.iterateFrom(tmpOffset);
|
|||
|
|
+ CqUnit cqUnit = null;
|
|||
|
|
+ if (null == iterator || (cqUnit = iterator.next()) == null) {
|
|||
|
|
+ // offset in msg may be greater than offset of cq.
|
|||
|
|
+ tmpOffset -= 1;
|
|||
|
|
+ continue;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ long offsetPyTemp = cqUnit.getPos();
|
|||
|
|
+ int sizePyTemp = cqUnit.getSize();
|
|||
|
|
if (offsetPyTemp == offsetPy && sizePyTemp == sizePy) {
|
|||
|
|
LOGGER.info("reviseQueueOffset check cq offset ok. {}, {}, {}",
|
|||
|
|
tmpOffset, offsetPyTemp, sizePyTemp);
|
|||
|
|
@@ -365,7 +370,9 @@ public class TimerMessageStore {
|
|||
|
|
} catch (Throwable e) {
|
|||
|
|
LOGGER.error("reviseQueueOffset check cq offset error.", e);
|
|||
|
|
} finally {
|
|||
|
|
- bufferCQ.release();
|
|||
|
|
+ if (iterator != null) {
|
|||
|
|
+ iterator.release();
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@@ -633,7 +640,7 @@ public class TimerMessageStore {
|
|||
|
|
if (!isRunningEnqueue()) {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
- ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId);
|
|||
|
|
+ ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId);
|
|||
|
|
if (null == cq) {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
@@ -643,18 +650,22 @@ public class TimerMessageStore {
|
|||
|
|
currQueueOffset = cq.getMinOffsetInQueue();
|
|||
|
|
}
|
|||
|
|
long offset = currQueueOffset;
|
|||
|
|
- SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(offset);
|
|||
|
|
- if (null == bufferCQ) {
|
|||
|
|
- return false;
|
|||
|
|
- }
|
|||
|
|
+ ReferredIterator<CqUnit> iterator = null;
|
|||
|
|
try {
|
|||
|
|
+ iterator = cq.iterateFrom(offset);
|
|||
|
|
+ if (null == iterator) {
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
int i = 0;
|
|||
|
|
- for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
|
|||
|
|
+ while (iterator.hasNext()) {
|
|||
|
|
+ i++;
|
|||
|
|
perfCounterTicks.startTick("enqueue_get");
|
|||
|
|
try {
|
|||
|
|
- long offsetPy = bufferCQ.getByteBuffer().getLong();
|
|||
|
|
- int sizePy = bufferCQ.getByteBuffer().getInt();
|
|||
|
|
- bufferCQ.getByteBuffer().getLong(); //tags code
|
|||
|
|
+ CqUnit cqUnit = iterator.next();
|
|||
|
|
+ long offsetPy = cqUnit.getPos();
|
|||
|
|
+ int sizePy = cqUnit.getSize();
|
|||
|
|
+ cqUnit.getTagsCode(); //tags code
|
|||
|
|
MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy);
|
|||
|
|
if (null == msgExt) {
|
|||
|
|
perfCounterTicks.getCounter("enqueue_get_miss");
|
|||
|
|
@@ -663,7 +674,7 @@ public class TimerMessageStore {
|
|||
|
|
lastEnqueueButExpiredStoreTime = msgExt.getStoreTimestamp();
|
|||
|
|
long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS));
|
|||
|
|
// use CQ offset, not offset in Message
|
|||
|
|
- msgExt.setQueueOffset(offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE));
|
|||
|
|
+ msgExt.setQueueOffset(offset + i);
|
|||
|
|
TimerRequest timerRequest = new TimerRequest(offsetPy, sizePy, delayedTime, System.currentTimeMillis(), MAGIC_DEFAULT, msgExt);
|
|||
|
|
// System.out.printf("build enqueue request, %s%n", timerRequest);
|
|||
|
|
while (!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) {
|
|||
|
|
@@ -687,14 +698,16 @@ public class TimerMessageStore {
|
|||
|
|
if (!isRunningEnqueue()) {
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
- currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE);
|
|||
|
|
+ currQueueOffset = offset + i;
|
|||
|
|
}
|
|||
|
|
- currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE);
|
|||
|
|
+ currQueueOffset = offset + i;
|
|||
|
|
return i > 0;
|
|||
|
|
} catch (Exception e) {
|
|||
|
|
LOGGER.error("Unknown exception in enqueuing", e);
|
|||
|
|
} finally {
|
|||
|
|
- bufferCQ.release();
|
|||
|
|
+ if (iterator != null) {
|
|||
|
|
+ iterator.release();
|
|||
|
|
+ }
|
|||
|
|
}
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
@@ -1642,7 +1655,7 @@ public class TimerMessageStore {
|
|||
|
|
if (System.currentTimeMillis() - start > storeConfig.getTimerProgressLogIntervalMs()) {
|
|||
|
|
start = System.currentTimeMillis();
|
|||
|
|
long tmpQueueOffset = currQueueOffset;
|
|||
|
|
- ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0);
|
|||
|
|
+ ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0);
|
|||
|
|
long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue();
|
|||
|
|
TimerMessageStore.LOGGER.info("[{}]Timer progress-check commitRead:[{}] currRead:[{}] currWrite:[{}] readBehind:{} currReadOffset:{} offsetBehind:{} behindMaster:{} " +
|
|||
|
|
"enqPutQueue:{} deqGetQueue:{} deqPutQueue:{} allCongestNum:{} enqExpiredStoreTime:{}",
|
|||
|
|
@@ -1685,7 +1698,7 @@ public class TimerMessageStore {
|
|||
|
|
|
|||
|
|
public long getEnqueueBehindMessages() {
|
|||
|
|
long tmpQueueOffset = currQueueOffset;
|
|||
|
|
- ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0);
|
|||
|
|
+ ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0);
|
|||
|
|
long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue();
|
|||
|
|
return maxOffsetInQueue - tmpQueueOffset;
|
|||
|
|
}
|
|||
|
|
diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
|
|||
|
|
index 12d1e5723..1d09ca86e 100644
|
|||
|
|
--- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
|
|||
|
|
+++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
|
|||
|
|
@@ -428,9 +428,10 @@ public class DefaultMessageStoreTest {
|
|||
|
|
|
|||
|
|
private long getStoreTime(CqUnit cqUnit) {
|
|||
|
|
try {
|
|||
|
|
- Method getStoreTime = getDefaultMessageStore().getClass().getDeclaredMethod("getStoreTime", CqUnit.class);
|
|||
|
|
+ Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass();
|
|||
|
|
+ Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class);
|
|||
|
|
getStoreTime.setAccessible(true);
|
|||
|
|
- return (long) getStoreTime.invoke(getDefaultMessageStore(), cqUnit);
|
|||
|
|
+ return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit);
|
|||
|
|
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
|
|||
|
|
throw new RuntimeException(e);
|
|||
|
|
}
|
|||
|
|
diff --git a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
|
|||
|
|
index 85626a332..2447bbf68 100644
|
|||
|
|
--- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
|
|||
|
|
+++ b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
|
|||
|
|
@@ -28,9 +28,11 @@ import org.apache.rocketmq.common.message.MessageConst;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageDecoder;
|
|||
|
|
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
+import org.apache.rocketmq.store.queue.MultiDispatch;
|
|||
|
|
import org.junit.After;
|
|||
|
|
import org.junit.Before;
|
|||
|
|
import org.junit.Test;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
|
|||
|
|
import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue;
|
|||
|
|
import static org.junit.Assert.assertEquals;
|
|||
|
|
@@ -69,15 +71,15 @@ public class MultiDispatchTest {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Test
|
|||
|
|
- public void queueKey() {
|
|||
|
|
+ public void lmqQueueKey() {
|
|||
|
|
MessageExtBrokerInner messageExtBrokerInner = mock(MessageExtBrokerInner.class);
|
|||
|
|
when(messageExtBrokerInner.getQueueId()).thenReturn(2);
|
|||
|
|
- String ret = consumeQueue.queueKey("%LMQ%lmq123", messageExtBrokerInner);
|
|||
|
|
+ String ret = MultiDispatch.lmqQueueKey("%LMQ%lmq123");
|
|||
|
|
assertEquals(ret, "%LMQ%lmq123-0");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Test
|
|||
|
|
- public void wrapMultiDispatch() {
|
|||
|
|
+ public void wrapMultiDispatch() throws RocksDBException {
|
|||
|
|
MessageExtBrokerInner messageExtBrokerInner = buildMessageMultiQueue();
|
|||
|
|
messageStore.assignOffset(messageExtBrokerInner);
|
|||
|
|
assertEquals(messageExtBrokerInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), "0,0");
|
|||
|
|
diff --git a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..acf5edf51
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java
|
|||
|
|
@@ -0,0 +1,1060 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+
|
|||
|
|
+package org.apache.rocketmq.store;
|
|||
|
|
+
|
|||
|
|
+import java.io.File;
|
|||
|
|
+import java.io.RandomAccessFile;
|
|||
|
|
+import java.lang.reflect.InvocationTargetException;
|
|||
|
|
+import java.lang.reflect.Method;
|
|||
|
|
+import java.net.InetAddress;
|
|||
|
|
+import java.net.InetSocketAddress;
|
|||
|
|
+import java.net.SocketAddress;
|
|||
|
|
+import java.net.UnknownHostException;
|
|||
|
|
+import java.nio.MappedByteBuffer;
|
|||
|
|
+import java.nio.channels.FileChannel;
|
|||
|
|
+import java.nio.channels.OverlappingFileLockException;
|
|||
|
|
+import java.nio.charset.StandardCharsets;
|
|||
|
|
+import java.util.ArrayList;
|
|||
|
|
+import java.util.Arrays;
|
|||
|
|
+import java.util.HashSet;
|
|||
|
|
+import java.util.List;
|
|||
|
|
+import java.util.Map;
|
|||
|
|
+import java.util.Random;
|
|||
|
|
+import java.util.UUID;
|
|||
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|||
|
|
+import java.util.concurrent.ConcurrentMap;
|
|||
|
|
+import java.util.concurrent.TimeUnit;
|
|||
|
|
+import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
|
+
|
|||
|
|
+import com.google.common.collect.Sets;
|
|||
|
|
+
|
|||
|
|
+import org.apache.rocketmq.common.BrokerConfig;
|
|||
|
|
+import org.apache.rocketmq.common.MixAll;
|
|||
|
|
+import org.apache.rocketmq.common.TopicConfig;
|
|||
|
|
+import org.apache.rocketmq.common.UtilAll;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageBatch;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageDecoder;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageExt;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageExtBatch;
|
|||
|
|
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
|
|||
|
|
+import org.apache.rocketmq.store.config.BrokerRole;
|
|||
|
|
+import org.apache.rocketmq.store.config.FlushDiskType;
|
|||
|
|
+import org.apache.rocketmq.store.config.MessageStoreConfig;
|
|||
|
|
+import org.apache.rocketmq.store.config.StorePathConfigHelper;
|
|||
|
|
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
|
|||
|
|
+import org.apache.rocketmq.store.queue.CqUnit;
|
|||
|
|
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
|
|||
|
|
+import org.assertj.core.util.Strings;
|
|||
|
|
+import org.junit.After;
|
|||
|
|
+import org.junit.Assert;
|
|||
|
|
+import org.junit.Before;
|
|||
|
|
+import org.junit.Test;
|
|||
|
|
+import org.junit.runner.RunWith;
|
|||
|
|
+import org.mockito.junit.MockitoJUnitRunner;
|
|||
|
|
+
|
|||
|
|
+import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
+import static org.junit.Assert.assertEquals;
|
|||
|
|
+import static org.junit.Assert.assertTrue;
|
|||
|
|
+
|
|||
|
|
+@RunWith(MockitoJUnitRunner.class)
|
|||
|
|
+public class RocksDBMessageStoreTest {
|
|||
|
|
+ private final String storeMessage = "Once, there was a chance for me!";
|
|||
|
|
+ private final String messageTopic = "FooBar";
|
|||
|
|
+ private final String storeType = StoreType.DEFAULT_ROCKSDB.getStoreType();
|
|||
|
|
+ private int queueTotal = 100;
|
|||
|
|
+ private AtomicInteger queueId = new AtomicInteger(0);
|
|||
|
|
+ private SocketAddress bornHost;
|
|||
|
|
+ private SocketAddress storeHost;
|
|||
|
|
+ private byte[] messageBody;
|
|||
|
|
+ private MessageStore messageStore;
|
|||
|
|
+
|
|||
|
|
+ @Before
|
|||
|
|
+ public void init() throws Exception {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);
|
|||
|
|
+ bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0);
|
|||
|
|
+
|
|||
|
|
+ messageStore = buildMessageStore();
|
|||
|
|
+ boolean load = messageStore.load();
|
|||
|
|
+ assertTrue(load);
|
|||
|
|
+ messageStore.start();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test(expected = OverlappingFileLockException.class)
|
|||
|
|
+ public void test_repeat_restart() throws Exception {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ throw new OverlappingFileLockException();
|
|||
|
|
+ }
|
|||
|
|
+ queueTotal = 1;
|
|||
|
|
+ messageBody = storeMessage.getBytes();
|
|||
|
|
+
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
|
|||
|
|
+ messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);
|
|||
|
|
+ messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4);
|
|||
|
|
+ messageStoreConfig.setMaxHashSlotNum(100);
|
|||
|
|
+ messageStoreConfig.setMaxIndexNum(100 * 10);
|
|||
|
|
+ messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir") + File.separator + "store");
|
|||
|
|
+ messageStoreConfig.setHaListenPort(0);
|
|||
|
|
+ MessageStore master = new RocksDBMessageStore(messageStoreConfig, null, new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>());
|
|||
|
|
+
|
|||
|
|
+ boolean load = master.load();
|
|||
|
|
+ assertTrue(load);
|
|||
|
|
+
|
|||
|
|
+ try {
|
|||
|
|
+ master.start();
|
|||
|
|
+ master.start();
|
|||
|
|
+ } finally {
|
|||
|
|
+ master.shutdown();
|
|||
|
|
+ master.destroy();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @After
|
|||
|
|
+ public void destroy() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ messageStore.shutdown();
|
|||
|
|
+ messageStore.destroy();
|
|||
|
|
+
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
|
|||
|
|
+ File file = new File(messageStoreConfig.getStorePathRootDir());
|
|||
|
|
+ UtilAll.deleteFile(file);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private MessageStore buildMessageStore() throws Exception {
|
|||
|
|
+ return buildMessageStore(null, "");
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private MessageStore buildMessageStore(String storePathRootDir, String topic) throws Exception {
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
|
|||
|
|
+ messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);
|
|||
|
|
+ messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);
|
|||
|
|
+ messageStoreConfig.setMaxHashSlotNum(10000);
|
|||
|
|
+ messageStoreConfig.setMaxIndexNum(100 * 100);
|
|||
|
|
+ messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
|
|||
|
|
+ messageStoreConfig.setFlushIntervalConsumeQueue(1);
|
|||
|
|
+ messageStoreConfig.setStoreType(storeType);
|
|||
|
|
+ messageStoreConfig.setHaListenPort(0);
|
|||
|
|
+ if (Strings.isNullOrEmpty(storePathRootDir)) {
|
|||
|
|
+ UUID uuid = UUID.randomUUID();
|
|||
|
|
+ storePathRootDir = System.getProperty("java.io.tmpdir") + File.separator + "store-" + uuid.toString();
|
|||
|
|
+ }
|
|||
|
|
+ messageStoreConfig.setStorePathRootDir(storePathRootDir);
|
|||
|
|
+ ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();
|
|||
|
|
+ topicConfigTable.put(topic, new TopicConfig(topic, 1, 1));
|
|||
|
|
+ return new RocksDBMessageStore(messageStoreConfig,
|
|||
|
|
+ new BrokerStatsManager("simpleTest", true),
|
|||
|
|
+ new MyMessageArrivingListener(),
|
|||
|
|
+ new BrokerConfig(), topicConfigTable);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testWriteAndRead() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ long ipv4HostMsgs = 10;
|
|||
|
|
+ long ipv6HostMsgs = 10;
|
|||
|
|
+ long totalMsgs = ipv4HostMsgs + ipv6HostMsgs;
|
|||
|
|
+ queueTotal = 1;
|
|||
|
|
+ messageBody = storeMessage.getBytes();
|
|||
|
|
+ for (long i = 0; i < ipv4HostMsgs; i++) {
|
|||
|
|
+ messageStore.putMessage(buildMessage());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ for (long i = 0; i < ipv6HostMsgs; i++) {
|
|||
|
|
+ messageStore.putMessage(buildIPv6HostMessage());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+
|
|||
|
|
+ for (long i = 0; i < totalMsgs; i++) {
|
|||
|
|
+ GetMessageResult result = messageStore.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null);
|
|||
|
|
+ assertThat(result).isNotNull();
|
|||
|
|
+ result.release();
|
|||
|
|
+ }
|
|||
|
|
+ verifyThatMasterIsFunctional(totalMsgs, messageStore);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testLookMessageByOffset_OffsetIsFirst() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = new Random().nextInt(10);
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ int firstOffset = 0;
|
|||
|
|
+ AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId);
|
|||
|
|
+ AppendMessageResult firstResult = appendMessageResultArray[0];
|
|||
|
|
+
|
|||
|
|
+ MessageExt messageExt = messageStore.lookMessageByOffset(firstResult.getWroteOffset());
|
|||
|
|
+ MessageExt messageExt1 = getDefaultMessageStore().lookMessageByOffset(firstResult.getWroteOffset(), firstResult.getWroteBytes());
|
|||
|
|
+
|
|||
|
|
+ assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset));
|
|||
|
|
+ assertThat(new String(messageExt1.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testLookMessageByOffset_OffsetIsLast() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = new Random().nextInt(10);
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId);
|
|||
|
|
+ int lastIndex = totalCount - 1;
|
|||
|
|
+ AppendMessageResult lastResult = appendMessageResultArray[lastIndex];
|
|||
|
|
+
|
|||
|
|
+ MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastResult.getWroteOffset(), lastResult.getWroteBytes());
|
|||
|
|
+
|
|||
|
|
+ assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, lastIndex));
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testLookMessageByOffset_OffsetIsOutOfBound() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = new Random().nextInt(10);
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId);
|
|||
|
|
+ long lastOffset = getMaxOffset(appendMessageResultArray);
|
|||
|
|
+
|
|||
|
|
+ MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastOffset);
|
|||
|
|
+
|
|||
|
|
+ assertThat(messageExt).isNull();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetOffsetInQueueByTime() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true);
|
|||
|
|
+ //Thread.sleep(10);
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+
|
|||
|
|
+ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);
|
|||
|
|
+ for (AppendMessageResult appendMessageResult : appendMessageResults) {
|
|||
|
|
+ long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp());
|
|||
|
|
+ CqUnit cqUnit = consumeQueue.get(offset);
|
|||
|
|
+ assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset());
|
|||
|
|
+ assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetOffsetInQueueByTime_TimestampIsSkewing() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true);
|
|||
|
|
+ //Thread.sleep(10);
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+ int skewing = 2;
|
|||
|
|
+
|
|||
|
|
+ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);
|
|||
|
|
+ for (AppendMessageResult appendMessageResult : appendMessageResults) {
|
|||
|
|
+ long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing);
|
|||
|
|
+ CqUnit cqUnit = consumeQueue.get(offset);
|
|||
|
|
+ assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset());
|
|||
|
|
+ assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetOffsetInQueueByTime_TimestampSkewingIsLarge() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true);
|
|||
|
|
+ //Thread.sleep(10);
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+ int skewing = 20000;
|
|||
|
|
+
|
|||
|
|
+ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);
|
|||
|
|
+ for (AppendMessageResult appendMessageResult : appendMessageResults) {
|
|||
|
|
+ long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing);
|
|||
|
|
+ CqUnit cqUnit = consumeQueue.get(offset);
|
|||
|
|
+ assertThat(cqUnit.getPos()).isEqualTo(appendMessageResults[0].getWroteOffset());
|
|||
|
|
+ assertThat(cqUnit.getSize()).isEqualTo(appendMessageResults[0].getWroteBytes());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetOffsetInQueueByTime_ConsumeQueueNotFound1() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ int wrongQueueId = 1;
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false);
|
|||
|
|
+ //Thread.sleep(10);
|
|||
|
|
+
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+
|
|||
|
|
+ long offset = messageStore.getOffsetInQueueByTime(topic, wrongQueueId, appendMessageResults[0].getStoreTimestamp());
|
|||
|
|
+
|
|||
|
|
+ assertThat(offset).isEqualTo(0);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetOffsetInQueueByTime_ConsumeQueueNotFound2() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ int wrongQueueId = 1;
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ putMessages(totalCount, topic, queueId, false);
|
|||
|
|
+ //Thread.sleep(10);
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+
|
|||
|
|
+ long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, 0);
|
|||
|
|
+
|
|||
|
|
+ assertThat(messageStoreTimeStamp).isEqualTo(-1);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetOffsetInQueueByTime_ConsumeQueueOffsetNotExist() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ int wrongQueueId = 1;
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ putMessages(totalCount, topic, queueId, true);
|
|||
|
|
+ //Thread.sleep(10);
|
|||
|
|
+
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+
|
|||
|
|
+ long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, -1);
|
|||
|
|
+
|
|||
|
|
+ assertThat(messageStoreTimeStamp).isEqualTo(-1);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetMessageStoreTimeStamp() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false);
|
|||
|
|
+ //Thread.sleep(10);
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+
|
|||
|
|
+ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);
|
|||
|
|
+ int minOffsetInQueue = (int) consumeQueue.getMinOffsetInQueue();
|
|||
|
|
+ for (int i = minOffsetInQueue; i < consumeQueue.getMaxOffsetInQueue(); i++) {
|
|||
|
|
+ long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, queueId, i);
|
|||
|
|
+ assertThat(messageStoreTimeStamp).isEqualTo(appendMessageResults[i].getStoreTimestamp());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetStoreTime_ParamIsNull() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ long storeTime = getStoreTime(null);
|
|||
|
|
+
|
|||
|
|
+ assertThat(storeTime).isEqualTo(-1);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetStoreTime_EverythingIsOk() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ final int totalCount = 10;
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ String topic = "FooBar";
|
|||
|
|
+ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false);
|
|||
|
|
+ //Thread.sleep(10);
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+ ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, queueId);
|
|||
|
|
+
|
|||
|
|
+ for (int i = 0; i < totalCount; i++) {
|
|||
|
|
+ CqUnit cqUnit = consumeQueue.get(i);
|
|||
|
|
+ long storeTime = getStoreTime(cqUnit);
|
|||
|
|
+ assertThat(storeTime).isEqualTo(appendMessageResults[i].getStoreTimestamp());
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ long phyOffset = -10;
|
|||
|
|
+ int size = 138;
|
|||
|
|
+ CqUnit cqUnit = new CqUnit(0, phyOffset, size, 0);
|
|||
|
|
+ long storeTime = getStoreTime(cqUnit);
|
|||
|
|
+
|
|||
|
|
+ assertThat(storeTime).isEqualTo(-1);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testPutMessage_whenMessagePropertyIsTooLong() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ String topicName = "messagePropertyIsTooLongTest";
|
|||
|
|
+ MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1);
|
|||
|
|
+ assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);
|
|||
|
|
+ assertEquals(0L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue());
|
|||
|
|
+ MessageExtBrokerInner normalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, 100);
|
|||
|
|
+ assertEquals(messageStore.putMessage(normalMessage).getPutMessageStatus(), PutMessageStatus.PUT_OK);
|
|||
|
|
+ assertEquals(1L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private RocksDBMessageStore getDefaultMessageStore() {
|
|||
|
|
+ return (RocksDBMessageStore) this.messageStore;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId) {
|
|||
|
|
+ return putMessages(totalCount, topic, queueId, false);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId, boolean interval) {
|
|||
|
|
+ AppendMessageResult[] appendMessageResultArray = new AppendMessageResult[totalCount];
|
|||
|
|
+ for (int i = 0; i < totalCount; i++) {
|
|||
|
|
+ String messageBody = buildMessageBodyByOffset(storeMessage, i);
|
|||
|
|
+
|
|||
|
|
+ MessageExtBrokerInner msgInner =
|
|||
|
|
+ i < totalCount / 2 ? buildMessage(messageBody.getBytes(), topic) : buildIPv6HostMessage(messageBody.getBytes(), topic);
|
|||
|
|
+ msgInner.setQueueId(queueId);
|
|||
|
|
+ PutMessageResult result = messageStore.putMessage(msgInner);
|
|||
|
|
+ appendMessageResultArray[i] = result.getAppendMessageResult();
|
|||
|
|
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);
|
|||
|
|
+ if (interval) {
|
|||
|
|
+ try {
|
|||
|
|
+ Thread.sleep(10);
|
|||
|
|
+ } catch (InterruptedException e) {
|
|||
|
|
+ throw new RuntimeException("Thread sleep ERROR");
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ return appendMessageResultArray;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private long getMaxOffset(AppendMessageResult[] appendMessageResultArray) {
|
|||
|
|
+ if (appendMessageResultArray == null) {
|
|||
|
|
+ return 0;
|
|||
|
|
+ }
|
|||
|
|
+ AppendMessageResult last = appendMessageResultArray[appendMessageResultArray.length - 1];
|
|||
|
|
+ return last.getWroteOffset() + last.getWroteBytes();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private String buildMessageBodyByOffset(String message, long i) {
|
|||
|
|
+ return String.format("%s offset %d", message, i);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private long getStoreTime(CqUnit cqUnit) {
|
|||
|
|
+ try {
|
|||
|
|
+ Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass();
|
|||
|
|
+ Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class);
|
|||
|
|
+ getStoreTime.setAccessible(true);
|
|||
|
|
+ return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit);
|
|||
|
|
+ } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
|
|||
|
|
+ throw new RuntimeException(e);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private MessageExtBrokerInner buildMessage(byte[] messageBody, String topic) {
|
|||
|
|
+ MessageExtBrokerInner msg = new MessageExtBrokerInner();
|
|||
|
|
+ msg.setTopic(topic);
|
|||
|
|
+ msg.setTags("TAG1");
|
|||
|
|
+ msg.setKeys("Hello");
|
|||
|
|
+ msg.setBody(messageBody);
|
|||
|
|
+ msg.setKeys(String.valueOf(System.currentTimeMillis()));
|
|||
|
|
+ msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);
|
|||
|
|
+ msg.setSysFlag(0);
|
|||
|
|
+ msg.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
+ msg.setStoreHost(storeHost);
|
|||
|
|
+ msg.setBornHost(bornHost);
|
|||
|
|
+ msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
|
|||
|
|
+ return msg;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private MessageExtBrokerInner buildSpecifyLengthPropertyMessage(byte[] messageBody, String topic, int length) {
|
|||
|
|
+ StringBuilder stringBuilder = new StringBuilder();
|
|||
|
|
+ Random random = new Random();
|
|||
|
|
+ for (int i = 0; i < length; i++) {
|
|||
|
|
+ stringBuilder.append(random.nextInt(10));
|
|||
|
|
+ }
|
|||
|
|
+ MessageExtBrokerInner msg = new MessageExtBrokerInner();
|
|||
|
|
+ msg.putUserProperty("test", stringBuilder.toString());
|
|||
|
|
+ msg.setTopic(topic);
|
|||
|
|
+ msg.setTags("TAG1");
|
|||
|
|
+ msg.setKeys("Hello");
|
|||
|
|
+ msg.setBody(messageBody);
|
|||
|
|
+ msg.setQueueId(0);
|
|||
|
|
+ msg.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
+ msg.setStoreHost(storeHost);
|
|||
|
|
+ msg.setBornHost(bornHost);
|
|||
|
|
+ msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
|
|||
|
|
+ return msg;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private MessageExtBrokerInner buildIPv6HostMessage(byte[] messageBody, String topic) {
|
|||
|
|
+ MessageExtBrokerInner msg = new MessageExtBrokerInner();
|
|||
|
|
+ msg.setTopic(topic);
|
|||
|
|
+ msg.setTags("TAG1");
|
|||
|
|
+ msg.setKeys("Hello");
|
|||
|
|
+ msg.setBody(messageBody);
|
|||
|
|
+ msg.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0");
|
|||
|
|
+ msg.setKeys(String.valueOf(System.currentTimeMillis()));
|
|||
|
|
+ msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);
|
|||
|
|
+ msg.setSysFlag(0);
|
|||
|
|
+ msg.setBornHostV6Flag();
|
|||
|
|
+ msg.setStoreHostAddressV6Flag();
|
|||
|
|
+ msg.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
+ try {
|
|||
|
|
+ msg.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 0));
|
|||
|
|
+ } catch (UnknownHostException e) {
|
|||
|
|
+ e.printStackTrace();
|
|||
|
|
+ assertThat(Boolean.FALSE).isTrue();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ try {
|
|||
|
|
+ msg.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 0));
|
|||
|
|
+ } catch (UnknownHostException e) {
|
|||
|
|
+ e.printStackTrace();
|
|||
|
|
+ assertThat(Boolean.FALSE).isTrue();
|
|||
|
|
+ }
|
|||
|
|
+ return msg;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private MessageExtBrokerInner buildMessage() {
|
|||
|
|
+ return buildMessage(messageBody, messageTopic);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ public MessageExtBatch buildMessageBatch(MessageBatch msgBatch) {
|
|||
|
|
+ MessageExtBatch msgExtBatch = new MessageExtBatch();
|
|||
|
|
+ msgExtBatch.setTopic(messageTopic);
|
|||
|
|
+ msgExtBatch.setTags("TAG1");
|
|||
|
|
+ msgExtBatch.setKeys("Hello");
|
|||
|
|
+ msgExtBatch.setBody(msgBatch.getBody());
|
|||
|
|
+ msgExtBatch.setKeys(String.valueOf(System.currentTimeMillis()));
|
|||
|
|
+ msgExtBatch.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);
|
|||
|
|
+ msgExtBatch.setSysFlag(0);
|
|||
|
|
+ msgExtBatch.setBornTimestamp(System.currentTimeMillis());
|
|||
|
|
+ msgExtBatch.setStoreHost(storeHost);
|
|||
|
|
+ msgExtBatch.setBornHost(bornHost);
|
|||
|
|
+ return msgExtBatch;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGroupCommit() throws Exception {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ long totalMsgs = 10;
|
|||
|
|
+ queueTotal = 1;
|
|||
|
|
+ messageBody = storeMessage.getBytes();
|
|||
|
|
+ for (long i = 0; i < totalMsgs; i++) {
|
|||
|
|
+ messageStore.putMessage(buildMessage());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ for (long i = 0; i < totalMsgs; i++) {
|
|||
|
|
+ GetMessageResult result = messageStore.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null);
|
|||
|
|
+ assertThat(result).isNotNull();
|
|||
|
|
+ result.release();
|
|||
|
|
+ }
|
|||
|
|
+ verifyThatMasterIsFunctional(totalMsgs, messageStore);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testMaxOffset() throws InterruptedException {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ int firstBatchMessages = 3;
|
|||
|
|
+ int queueId = 0;
|
|||
|
|
+ messageBody = storeMessage.getBytes();
|
|||
|
|
+
|
|||
|
|
+ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(0);
|
|||
|
|
+
|
|||
|
|
+ for (int i = 0; i < firstBatchMessages; i++) {
|
|||
|
|
+ final MessageExtBrokerInner msg = buildMessage();
|
|||
|
|
+ msg.setQueueId(queueId);
|
|||
|
|
+ messageStore.putMessage(msg);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ while (messageStore.dispatchBehindBytes() != 0) {
|
|||
|
|
+ TimeUnit.MILLISECONDS.sleep(1);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages);
|
|||
|
|
+
|
|||
|
|
+ // Disable the dispatcher
|
|||
|
|
+ messageStore.getDispatcherList().clear();
|
|||
|
|
+
|
|||
|
|
+ int secondBatchMessages = 2;
|
|||
|
|
+
|
|||
|
|
+ for (int i = 0; i < secondBatchMessages; i++) {
|
|||
|
|
+ final MessageExtBrokerInner msg = buildMessage();
|
|||
|
|
+ msg.setQueueId(queueId);
|
|||
|
|
+ messageStore.putMessage(msg);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages);
|
|||
|
|
+ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, true)).isEqualTo(firstBatchMessages);
|
|||
|
|
+ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, false)).isEqualTo(firstBatchMessages + secondBatchMessages);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private MessageExtBrokerInner buildIPv6HostMessage() {
|
|||
|
|
+ return buildIPv6HostMessage(messageBody, "FooBar");
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void verifyThatMasterIsFunctional(long totalMsgs, MessageStore master) {
|
|||
|
|
+ for (long i = 0; i < totalMsgs; i++) {
|
|||
|
|
+ master.putMessage(buildMessage());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+
|
|||
|
|
+ for (long i = 0; i < totalMsgs; i++) {
|
|||
|
|
+ GetMessageResult result = master.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null);
|
|||
|
|
+ assertThat(result).isNotNull();
|
|||
|
|
+ result.release();
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testPullSize() throws Exception {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ String topic = "pullSizeTopic";
|
|||
|
|
+
|
|||
|
|
+ for (int i = 0; i < 32; i++) {
|
|||
|
|
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
|
|||
|
|
+ messageExtBrokerInner.setTopic(topic);
|
|||
|
|
+ messageExtBrokerInner.setQueueId(0);
|
|||
|
|
+ messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+ }
|
|||
|
|
+ // wait for consume queue build
|
|||
|
|
+ // the sleep time should be great than consume queue flush interval
|
|||
|
|
+ //Thread.sleep(100);
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+ String group = "simple";
|
|||
|
|
+ GetMessageResult getMessageResult32 = messageStore.getMessage(group, topic, 0, 0, 32, null);
|
|||
|
|
+ assertThat(getMessageResult32.getMessageBufferList().size()).isEqualTo(32);
|
|||
|
|
+ getMessageResult32.release();
|
|||
|
|
+
|
|||
|
|
+ GetMessageResult getMessageResult20 = messageStore.getMessage(group, topic, 0, 0, 20, null);
|
|||
|
|
+ assertThat(getMessageResult20.getMessageBufferList().size()).isEqualTo(20);
|
|||
|
|
+
|
|||
|
|
+ getMessageResult20.release();
|
|||
|
|
+ GetMessageResult getMessageResult45 = messageStore.getMessage(group, topic, 0, 0, 10, null);
|
|||
|
|
+ assertThat(getMessageResult45.getMessageBufferList().size()).isEqualTo(10);
|
|||
|
|
+ getMessageResult45.release();
|
|||
|
|
+
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testRecover() throws Exception {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ String topic = "recoverTopic";
|
|||
|
|
+ messageBody = storeMessage.getBytes();
|
|||
|
|
+ for (int i = 0; i < 100; i++) {
|
|||
|
|
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
|
|||
|
|
+ messageExtBrokerInner.setTopic(topic);
|
|||
|
|
+ messageExtBrokerInner.setQueueId(0);
|
|||
|
|
+ messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ // Thread.sleep(100);//wait for build consumer queue
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+
|
|||
|
|
+ long maxPhyOffset = messageStore.getMaxPhyOffset();
|
|||
|
|
+ long maxCqOffset = messageStore.getMaxOffsetInQueue(topic, 0);
|
|||
|
|
+
|
|||
|
|
+ //1.just reboot
|
|||
|
|
+ messageStore.shutdown();
|
|||
|
|
+ String storeRootDir = ((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir();
|
|||
|
|
+ messageStore = buildMessageStore(storeRootDir, topic);
|
|||
|
|
+ boolean load = messageStore.load();
|
|||
|
|
+ assertTrue(load);
|
|||
|
|
+ messageStore.start();
|
|||
|
|
+ assertTrue(maxPhyOffset == messageStore.getMaxPhyOffset());
|
|||
|
|
+ assertTrue(maxCqOffset == messageStore.getMaxOffsetInQueue(topic, 0));
|
|||
|
|
+
|
|||
|
|
+ //2.damage commit-log and reboot normal
|
|||
|
|
+ for (int i = 0; i < 100; i++) {
|
|||
|
|
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
|
|||
|
|
+ messageExtBrokerInner.setTopic(topic);
|
|||
|
|
+ messageExtBrokerInner.setQueueId(0);
|
|||
|
|
+ messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+ }
|
|||
|
|
+ //Thread.sleep(100);
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+ long secondLastPhyOffset = messageStore.getMaxPhyOffset();
|
|||
|
|
+ long secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0);
|
|||
|
|
+
|
|||
|
|
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
|
|||
|
|
+ messageExtBrokerInner.setTopic(topic);
|
|||
|
|
+ messageExtBrokerInner.setQueueId(0);
|
|||
|
|
+ messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+ messageStore.shutdown();
|
|||
|
|
+
|
|||
|
|
+ //damage last message
|
|||
|
|
+ damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset);
|
|||
|
|
+
|
|||
|
|
+ //reboot
|
|||
|
|
+ messageStore = buildMessageStore(storeRootDir, topic);
|
|||
|
|
+ load = messageStore.load();
|
|||
|
|
+ assertTrue(load);
|
|||
|
|
+ messageStore.start();
|
|||
|
|
+ assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset());
|
|||
|
|
+ assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0));
|
|||
|
|
+
|
|||
|
|
+ //3.damage commitlog and reboot abnormal
|
|||
|
|
+ for (int i = 0; i < 100; i++) {
|
|||
|
|
+ messageExtBrokerInner = buildMessage();
|
|||
|
|
+ messageExtBrokerInner.setTopic(topic);
|
|||
|
|
+ messageExtBrokerInner.setQueueId(0);
|
|||
|
|
+ messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+ }
|
|||
|
|
+ //Thread.sleep(100);
|
|||
|
|
+ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);
|
|||
|
|
+ secondLastPhyOffset = messageStore.getMaxPhyOffset();
|
|||
|
|
+ secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0);
|
|||
|
|
+
|
|||
|
|
+ messageExtBrokerInner = buildMessage();
|
|||
|
|
+ messageExtBrokerInner.setTopic(topic);
|
|||
|
|
+ messageExtBrokerInner.setQueueId(0);
|
|||
|
|
+ messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+ messageStore.shutdown();
|
|||
|
|
+
|
|||
|
|
+ //damage last message
|
|||
|
|
+ damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset);
|
|||
|
|
+ //add abort file
|
|||
|
|
+ String fileName = StorePathConfigHelper.getAbortFile(((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir());
|
|||
|
|
+ File file = new File(fileName);
|
|||
|
|
+ UtilAll.ensureDirOK(file.getParent());
|
|||
|
|
+ file.createNewFile();
|
|||
|
|
+
|
|||
|
|
+ messageStore = buildMessageStore(storeRootDir, topic);
|
|||
|
|
+ load = messageStore.load();
|
|||
|
|
+ assertTrue(load);
|
|||
|
|
+ messageStore.start();
|
|||
|
|
+ assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset());
|
|||
|
|
+ assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0));
|
|||
|
|
+
|
|||
|
|
+ //message write again
|
|||
|
|
+ for (int i = 0; i < 100; i++) {
|
|||
|
|
+ messageExtBrokerInner = buildMessage();
|
|||
|
|
+ messageExtBrokerInner.setTopic(topic);
|
|||
|
|
+ messageExtBrokerInner.setQueueId(0);
|
|||
|
|
+ messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testStorePathOK() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ if (messageStore instanceof RocksDBMessageStore) {
|
|||
|
|
+ assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathPhysic()));
|
|||
|
|
+ assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathLogic()));
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private boolean fileExists(String path) {
|
|||
|
|
+ if (path != null) {
|
|||
|
|
+ File f = new File(path);
|
|||
|
|
+ return f.exists();
|
|||
|
|
+ }
|
|||
|
|
+ return false;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private void damageCommitLog(RocksDBMessageStore store, long offset) throws Exception {
|
|||
|
|
+ assertThat(store).isNotNull();
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = store.getMessageStoreConfig();
|
|||
|
|
+ File file = new File(messageStoreConfig.getStorePathCommitLog() + File.separator + "00000000000000000000");
|
|||
|
|
+ try (RandomAccessFile raf = new RandomAccessFile(file, "rw");
|
|||
|
|
+ FileChannel fileChannel = raf.getChannel()) {
|
|||
|
|
+ MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, 1024 * 1024 * 10);
|
|||
|
|
+ int bodyLen = mappedByteBuffer.getInt((int) offset + 84);
|
|||
|
|
+ int topicLenIndex = (int) offset + 84 + bodyLen + 4;
|
|||
|
|
+ mappedByteBuffer.position(topicLenIndex);
|
|||
|
|
+ mappedByteBuffer.putInt(0);
|
|||
|
|
+ mappedByteBuffer.putInt(0);
|
|||
|
|
+ mappedByteBuffer.putInt(0);
|
|||
|
|
+ mappedByteBuffer.putInt(0);
|
|||
|
|
+ mappedByteBuffer.force();
|
|||
|
|
+ fileChannel.force(true);
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testPutMsgExceedsMaxLength() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ messageBody = new byte[4 * 1024 * 1024 + 1];
|
|||
|
|
+ MessageExtBrokerInner msg = buildMessage();
|
|||
|
|
+
|
|||
|
|
+ PutMessageResult result = messageStore.putMessage(msg);
|
|||
|
|
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.MESSAGE_ILLEGAL);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testPutMsgBatchExceedsMaxLength() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ messageBody = new byte[4 * 1024 * 1024 + 1];
|
|||
|
|
+ MessageExtBrokerInner msg1 = buildMessage();
|
|||
|
|
+ MessageExtBrokerInner msg2 = buildMessage();
|
|||
|
|
+ MessageExtBrokerInner msg3 = buildMessage();
|
|||
|
|
+
|
|||
|
|
+ MessageBatch msgBatch = MessageBatch.generateFromList(Arrays.asList(msg1, msg2, msg3));
|
|||
|
|
+ msgBatch.setBody(msgBatch.encode());
|
|||
|
|
+
|
|||
|
|
+ MessageExtBatch msgExtBatch = buildMessageBatch(msgBatch);
|
|||
|
|
+
|
|||
|
|
+ try {
|
|||
|
|
+ PutMessageResult result = this.messageStore.putMessages(msgExtBatch);
|
|||
|
|
+ } catch (Exception e) {
|
|||
|
|
+ assertThat(e.getMessage()).contains("message body size exceeded");
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testPutMsgWhenReplicasNotEnough() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig();
|
|||
|
|
+ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);
|
|||
|
|
+ messageStoreConfig.setTotalReplicas(2);
|
|||
|
|
+ messageStoreConfig.setInSyncReplicas(2);
|
|||
|
|
+ messageStoreConfig.setEnableAutoInSyncReplicas(false);
|
|||
|
|
+ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);
|
|||
|
|
+ this.messageStore.setAliveReplicaNumInGroup(1);
|
|||
|
|
+
|
|||
|
|
+ MessageExtBrokerInner msg = buildMessage();
|
|||
|
|
+ PutMessageResult result = this.messageStore.putMessage(msg);
|
|||
|
|
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH);
|
|||
|
|
+ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testPutMsgWhenAdaptiveDegradation() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig();
|
|||
|
|
+ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);
|
|||
|
|
+ messageStoreConfig.setTotalReplicas(2);
|
|||
|
|
+ messageStoreConfig.setInSyncReplicas(2);
|
|||
|
|
+ messageStoreConfig.setEnableAutoInSyncReplicas(true);
|
|||
|
|
+ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);
|
|||
|
|
+ this.messageStore.setAliveReplicaNumInGroup(1);
|
|||
|
|
+
|
|||
|
|
+ MessageExtBrokerInner msg = buildMessage();
|
|||
|
|
+ PutMessageResult result = this.messageStore.putMessage(msg);
|
|||
|
|
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);
|
|||
|
|
+ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);
|
|||
|
|
+ messageStoreConfig.setEnableAutoInSyncReplicas(false);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testGetBulkCommitLogData() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ RocksDBMessageStore defaultMessageStore = (RocksDBMessageStore) messageStore;
|
|||
|
|
+
|
|||
|
|
+ messageBody = new byte[2 * 1024 * 1024];
|
|||
|
|
+
|
|||
|
|
+ for (int i = 0; i < 10; i++) {
|
|||
|
|
+ MessageExtBrokerInner msg1 = buildMessage();
|
|||
|
|
+ messageStore.putMessage(msg1);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ List<SelectMappedBufferResult> bufferResultList = defaultMessageStore.getBulkCommitLogData(0, (int) defaultMessageStore.getMaxPhyOffset());
|
|||
|
|
+ List<MessageExt> msgList = new ArrayList<>();
|
|||
|
|
+ for (SelectMappedBufferResult bufferResult : bufferResultList) {
|
|||
|
|
+ msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false));
|
|||
|
|
+ bufferResult.release();
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ assertThat(msgList.size()).isEqualTo(10);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testPutLongMessage() throws Exception {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
|
|||
|
|
+ CommitLog commitLog = ((RocksDBMessageStore) messageStore).getCommitLog();
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig();
|
|||
|
|
+ MessageExtEncoder.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get();
|
|||
|
|
+
|
|||
|
|
+ //body size, topic size, properties size exactly equal to max size
|
|||
|
|
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);
|
|||
|
|
+ messageExtBrokerInner.setTopic(new String(new byte[127]));
|
|||
|
|
+ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));
|
|||
|
|
+ PutMessageResult encodeResult1 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
|
|||
|
|
+ assertTrue(encodeResult1 == null);
|
|||
|
|
+
|
|||
|
|
+ //body size exactly more than max message body size
|
|||
|
|
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]);
|
|||
|
|
+ PutMessageResult encodeResult2 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
|
|||
|
|
+ assertTrue(encodeResult2.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
|
|||
|
|
+
|
|||
|
|
+ //body size exactly equal to max message size
|
|||
|
|
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 64 * 1024]);
|
|||
|
|
+ PutMessageResult encodeResult3 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
|
|||
|
|
+ assertTrue(encodeResult3.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
|
|||
|
|
+
|
|||
|
|
+ //message properties length more than properties maxSize
|
|||
|
|
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);
|
|||
|
|
+ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE + 1]));
|
|||
|
|
+ PutMessageResult encodeResult4 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
|
|||
|
|
+ assertTrue(encodeResult4.getPutMessageStatus() == PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);
|
|||
|
|
+
|
|||
|
|
+ //message length more than buffer length capacity
|
|||
|
|
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);
|
|||
|
|
+ messageExtBrokerInner.setTopic(new String(new byte[Short.MAX_VALUE]));
|
|||
|
|
+ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));
|
|||
|
|
+ PutMessageResult encodeResult5 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
|
|||
|
|
+ assertTrue(encodeResult5.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testDynamicMaxMessageSize() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig();
|
|||
|
|
+ int originMaxMessageSize = messageStoreConfig.getMaxMessageSize();
|
|||
|
|
+
|
|||
|
|
+ messageExtBrokerInner.setBody(new byte[originMaxMessageSize + 10]);
|
|||
|
|
+ PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
|
|||
|
|
+
|
|||
|
|
+ int newMaxMessageSize = originMaxMessageSize + 10;
|
|||
|
|
+ messageStoreConfig.setMaxMessageSize(newMaxMessageSize);
|
|||
|
|
+ putMessageResult = messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK);
|
|||
|
|
+
|
|||
|
|
+ messageStoreConfig.setMaxMessageSize(10);
|
|||
|
|
+ putMessageResult = messageStore.putMessage(messageExtBrokerInner);
|
|||
|
|
+ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
|
|||
|
|
+
|
|||
|
|
+ messageStoreConfig.setMaxMessageSize(originMaxMessageSize);
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testDeleteTopics() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig();
|
|||
|
|
+ ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> consumeQueueTable =
|
|||
|
|
+ ((RocksDBMessageStore) messageStore).getConsumeQueueTable();
|
|||
|
|
+ for (int i = 0; i < 10; i++) {
|
|||
|
|
+ ConcurrentMap<Integer, ConsumeQueueInterface> cqTable = new ConcurrentHashMap<>();
|
|||
|
|
+ String topicName = "topic-" + i;
|
|||
|
|
+ for (int j = 0; j < 4; j++) {
|
|||
|
|
+ ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(),
|
|||
|
|
+ messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore);
|
|||
|
|
+ cqTable.put(j, consumeQueue);
|
|||
|
|
+ }
|
|||
|
|
+ consumeQueueTable.put(topicName, cqTable);
|
|||
|
|
+ }
|
|||
|
|
+ Assert.assertEquals(consumeQueueTable.size(), 10);
|
|||
|
|
+ HashSet<String> resultSet = Sets.newHashSet("topic-3", "topic-5");
|
|||
|
|
+ messageStore.deleteTopics(Sets.difference(consumeQueueTable.keySet(), resultSet));
|
|||
|
|
+ Assert.assertEquals(consumeQueueTable.size(), 2);
|
|||
|
|
+ Assert.assertEquals(resultSet, consumeQueueTable.keySet());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Test
|
|||
|
|
+ public void testCleanUnusedTopic() {
|
|||
|
|
+ if (notExecuted()) {
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig();
|
|||
|
|
+ ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> consumeQueueTable =
|
|||
|
|
+ ((RocksDBMessageStore) messageStore).getConsumeQueueTable();
|
|||
|
|
+ for (int i = 0; i < 10; i++) {
|
|||
|
|
+ ConcurrentMap<Integer, ConsumeQueueInterface> cqTable = new ConcurrentHashMap<>();
|
|||
|
|
+ String topicName = "topic-" + i;
|
|||
|
|
+ for (int j = 0; j < 4; j++) {
|
|||
|
|
+ ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(),
|
|||
|
|
+ messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore);
|
|||
|
|
+ cqTable.put(j, consumeQueue);
|
|||
|
|
+ }
|
|||
|
|
+ consumeQueueTable.put(topicName, cqTable);
|
|||
|
|
+ }
|
|||
|
|
+ Assert.assertEquals(consumeQueueTable.size(), 10);
|
|||
|
|
+ HashSet<String> resultSet = Sets.newHashSet("topic-3", "topic-5");
|
|||
|
|
+ messageStore.cleanUnusedTopic(resultSet);
|
|||
|
|
+ Assert.assertEquals(consumeQueueTable.size(), 2);
|
|||
|
|
+ Assert.assertEquals(resultSet, consumeQueueTable.keySet());
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private class MyMessageArrivingListener implements MessageArrivingListener {
|
|||
|
|
+ @Override
|
|||
|
|
+ public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,
|
|||
|
|
+ byte[] filterBitMap, Map<String, String> properties) {
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ private boolean notExecuted() {
|
|||
|
|
+ return MixAll.isMac();
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java
|
|||
|
|
index b2d99c3ed..17a2b5e19 100644
|
|||
|
|
--- a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java
|
|||
|
|
+++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java
|
|||
|
|
@@ -38,11 +38,16 @@ public class StoreTestUtil {
|
|||
|
|
|
|||
|
|
public static boolean isCommitLogAvailable(DefaultMessageStore store) {
|
|||
|
|
try {
|
|||
|
|
+ Field serviceField = null;
|
|||
|
|
+ if (store instanceof RocksDBMessageStore) {
|
|||
|
|
+ serviceField = store.getClass().getSuperclass().getDeclaredField("reputMessageService");
|
|||
|
|
+ } else {
|
|||
|
|
+ serviceField = store.getClass().getDeclaredField("reputMessageService");
|
|||
|
|
+ }
|
|||
|
|
|
|||
|
|
- Field serviceField = store.getClass().getDeclaredField("reputMessageService");
|
|||
|
|
serviceField.setAccessible(true);
|
|||
|
|
DefaultMessageStore.ReputMessageService reputService =
|
|||
|
|
- (DefaultMessageStore.ReputMessageService) serviceField.get(store);
|
|||
|
|
+ (DefaultMessageStore.ReputMessageService) serviceField.get(store);
|
|||
|
|
|
|||
|
|
Method method = DefaultMessageStore.ReputMessageService.class.getDeclaredMethod("isCommitLogAvailable");
|
|||
|
|
method.setAccessible(true);
|
|||
|
|
diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java
|
|||
|
|
index 54174ac16..fa8f41dbf 100644
|
|||
|
|
--- a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java
|
|||
|
|
+++ b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java
|
|||
|
|
@@ -36,7 +36,7 @@ import org.junit.Before;
|
|||
|
|
import org.junit.Test;
|
|||
|
|
import org.junit.runner.RunWith;
|
|||
|
|
import org.mockito.junit.MockitoJUnitRunner;
|
|||
|
|
-
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
import static org.assertj.core.api.Assertions.assertThat;
|
|||
|
|
import static org.awaitility.Awaitility.await;
|
|||
|
|
import static org.mockito.ArgumentMatchers.anyLong;
|
|||
|
|
@@ -114,7 +114,7 @@ public class HAServerTest {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Test
|
|||
|
|
- public void inSyncReplicasNums() throws IOException {
|
|||
|
|
+ public void inSyncReplicasNums() throws IOException, RocksDBException {
|
|||
|
|
DefaultMessageStore messageStore = mockMessageStore();
|
|||
|
|
doReturn(123L).when(messageStore).getMaxPhyOffset();
|
|||
|
|
doReturn(123L).when(messageStore).getMasterFlushedOffset();
|
|||
|
|
@@ -150,7 +150,7 @@ public class HAServerTest {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Test
|
|||
|
|
- public void isSlaveOK() throws IOException {
|
|||
|
|
+ public void isSlaveOK() throws IOException, RocksDBException {
|
|||
|
|
DefaultMessageStore messageStore = mockMessageStore();
|
|||
|
|
doReturn(123L).when(messageStore).getMaxPhyOffset();
|
|||
|
|
doReturn(123L).when(messageStore).getMasterFlushedOffset();
|
|||
|
|
@@ -175,7 +175,8 @@ public class HAServerTest {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Test
|
|||
|
|
- public void putRequest_SingleAck() throws IOException, ExecutionException, InterruptedException, TimeoutException {
|
|||
|
|
+ public void putRequest_SingleAck()
|
|||
|
|
+ throws IOException, ExecutionException, InterruptedException, TimeoutException, RocksDBException {
|
|||
|
|
CommitLog.GroupCommitRequest request = new CommitLog.GroupCommitRequest(124, 4000, 1);
|
|||
|
|
this.haService.putRequest(request);
|
|||
|
|
|
|||
|
|
@@ -192,7 +193,8 @@ public class HAServerTest {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Test
|
|||
|
|
- public void putRequest_MultipleAckAndRequests() throws IOException, ExecutionException, InterruptedException {
|
|||
|
|
+ public void putRequest_MultipleAckAndRequests()
|
|||
|
|
+ throws IOException, ExecutionException, InterruptedException, RocksDBException {
|
|||
|
|
CommitLog.GroupCommitRequest oneAck = new CommitLog.GroupCommitRequest(124, 4000, 2);
|
|||
|
|
this.haService.putRequest(oneAck);
|
|||
|
|
|
|||
|
|
@@ -218,7 +220,7 @@ public class HAServerTest {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@Test
|
|||
|
|
- public void getPush2SlaveMaxOffset() throws IOException {
|
|||
|
|
+ public void getPush2SlaveMaxOffset() throws IOException, RocksDBException {
|
|||
|
|
DefaultMessageStore messageStore = mockMessageStore();
|
|||
|
|
doReturn(123L).when(messageStore).getMaxPhyOffset();
|
|||
|
|
doReturn(123L).when(messageStore).getMasterFlushedOffset();
|
|||
|
|
@@ -256,7 +258,7 @@ public class HAServerTest {
|
|||
|
|
this.haClientList.add(haClient);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
- private DefaultMessageStore mockMessageStore() throws IOException {
|
|||
|
|
+ private DefaultMessageStore mockMessageStore() throws IOException, RocksDBException {
|
|||
|
|
DefaultMessageStore messageStore = mock(DefaultMessageStore.class);
|
|||
|
|
BrokerConfig brokerConfig = mock(BrokerConfig.class);
|
|||
|
|
|
|||
|
|
diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java
|
|||
|
|
index 27dcff141..db5c5af4c 100644
|
|||
|
|
--- a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java
|
|||
|
|
+++ b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java
|
|||
|
|
@@ -41,6 +41,7 @@ import org.junit.Assert;
|
|||
|
|
import org.junit.Assume;
|
|||
|
|
import org.junit.Ignore;
|
|||
|
|
import org.junit.Test;
|
|||
|
|
+import org.rocksdb.RocksDBException;
|
|||
|
|
|
|||
|
|
import java.io.File;
|
|||
|
|
import java.net.InetAddress;
|
|||
|
|
@@ -180,7 +181,7 @@ public class AutoSwitchHATest {
|
|||
|
|
|
|||
|
|
private boolean changeMasterAndPutMessage(DefaultMessageStore master, MessageStoreConfig masterConfig,
|
|||
|
|
DefaultMessageStore slave, long slaveId, MessageStoreConfig slaveConfig, int epoch, String masterHaAddress,
|
|||
|
|
- int totalPutMessageNums) {
|
|||
|
|
+ int totalPutMessageNums) throws RocksDBException {
|
|||
|
|
|
|||
|
|
boolean flag = true;
|
|||
|
|
// Change role
|
|||
|
|
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
index d7d13d61e..edaa5d19f 100644
|
|||
|
|
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
|
|||
|
|
@@ -16,17 +16,14 @@
|
|||
|
|
*/
|
|||
|
|
package org.apache.rocketmq.tieredstore;
|
|||
|
|
|
|||
|
|
-import com.google.common.base.Stopwatch;
|
|||
|
|
-import io.opentelemetry.api.common.Attributes;
|
|||
|
|
-import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
-import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
-import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
-import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
import java.util.List;
|
|||
|
|
import java.util.Set;
|
|||
|
|
import java.util.concurrent.CompletableFuture;
|
|||
|
|
import java.util.concurrent.TimeUnit;
|
|||
|
|
import java.util.function.Supplier;
|
|||
|
|
+
|
|||
|
|
+import com.google.common.base.Stopwatch;
|
|||
|
|
+
|
|||
|
|
import org.apache.commons.lang3.StringUtils;
|
|||
|
|
import org.apache.rocketmq.common.BoundaryType;
|
|||
|
|
import org.apache.rocketmq.common.MixAll;
|
|||
|
|
@@ -55,6 +52,12 @@ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant;
|
|||
|
|
import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;
|
|||
|
|
import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
|
|||
|
|
|
|||
|
|
+import io.opentelemetry.api.common.Attributes;
|
|||
|
|
+import io.opentelemetry.api.common.AttributesBuilder;
|
|||
|
|
+import io.opentelemetry.api.metrics.Meter;
|
|||
|
|
+import io.opentelemetry.sdk.metrics.InstrumentSelector;
|
|||
|
|
+import io.opentelemetry.sdk.metrics.ViewBuilder;
|
|||
|
|
+
|
|||
|
|
public class TieredMessageStore extends AbstractPluginMessageStore {
|
|||
|
|
|
|||
|
|
protected static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
|
|||
|
|
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java
|
|||
|
|
index 2a7d3fba4..1ecb1fa2c 100644
|
|||
|
|
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java
|
|||
|
|
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java
|
|||
|
|
@@ -106,8 +106,8 @@ public class ExportMetadataInRocksDBCommand implements SubCommand {
|
|||
|
|
final Map<String, JSONObject> jsonConfig = new HashMap<>();
|
|||
|
|
final Map<String, JSONObject> configTable = new HashMap<>();
|
|||
|
|
iterateKvStore(kvStore, (key, value) -> {
|
|||
|
|
- final String configKey = new String(key, DataConverter.charset);
|
|||
|
|
- final String configValue = new String(value, DataConverter.charset);
|
|||
|
|
+ final String configKey = new String(key, DataConverter.CHARSET_UTF8);
|
|||
|
|
+ final String configValue = new String(value, DataConverter.CHARSET_UTF8);
|
|||
|
|
final JSONObject jsonObject = JSONObject.parseObject(configValue);
|
|||
|
|
configTable.put(configKey, jsonObject);
|
|||
|
|
}
|
|||
|
|
@@ -120,8 +120,8 @@ public class ExportMetadataInRocksDBCommand implements SubCommand {
|
|||
|
|
} else {
|
|||
|
|
AtomicLong count = new AtomicLong(0);
|
|||
|
|
iterateKvStore(kvStore, (key, value) -> {
|
|||
|
|
- final String configKey = new String(key, DataConverter.charset);
|
|||
|
|
- final String configValue = new String(value, DataConverter.charset);
|
|||
|
|
+ final String configKey = new String(key, DataConverter.CHARSET_UTF8);
|
|||
|
|
+ final String configValue = new String(value, DataConverter.CHARSET_UTF8);
|
|||
|
|
System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), configKey, configValue);
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
new file mode 100644
|
|||
|
|
index 000000000..b987ad873
|
|||
|
|
--- /dev/null
|
|||
|
|
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
|
|||
|
|
@@ -0,0 +1,118 @@
|
|||
|
|
+/*
|
|||
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one or more
|
|||
|
|
+ * contributor license agreements. See the NOTICE file distributed with
|
|||
|
|
+ * this work for additional information regarding copyright ownership.
|
|||
|
|
+ * The ASF licenses this file to You under the Apache License, Version 2.0
|
|||
|
|
+ * (the "License"); you may not use this file except in compliance with
|
|||
|
|
+ * the License. You may obtain a copy of the License at
|
|||
|
|
+ *
|
|||
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
+ *
|
|||
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|||
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
+ * See the License for the specific language governing permissions and
|
|||
|
|
+ * limitations under the License.
|
|||
|
|
+ */
|
|||
|
|
+package org.apache.rocketmq.tools.command.metadata;
|
|||
|
|
+
|
|||
|
|
+import com.alibaba.fastjson.JSONObject;
|
|||
|
|
+import org.apache.commons.cli.CommandLine;
|
|||
|
|
+import org.apache.commons.cli.Option;
|
|||
|
|
+import org.apache.commons.cli.Options;
|
|||
|
|
+import org.apache.commons.lang3.StringUtils;
|
|||
|
|
+import org.apache.rocketmq.common.config.RocksDBConfigManager;
|
|||
|
|
+import org.apache.rocketmq.common.utils.DataConverter;
|
|||
|
|
+import org.apache.rocketmq.remoting.RPCHook;
|
|||
|
|
+import org.apache.rocketmq.tools.command.SubCommand;
|
|||
|
|
+import org.apache.rocketmq.tools.command.SubCommandException;
|
|||
|
|
+
|
|||
|
|
+import java.io.File;
|
|||
|
|
+import java.util.HashMap;
|
|||
|
|
+import java.util.Map;
|
|||
|
|
+
|
|||
|
|
+public class RocksDBConfigToJsonCommand implements SubCommand {
|
|||
|
|
+ private static final String TOPICS_JSON_CONFIG = "topics";
|
|||
|
|
+ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups";
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandName() {
|
|||
|
|
+ return "rocksDBConfigToJson";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public String commandDesc() {
|
|||
|
|
+ return "Convert RocksDB kv config (topics/subscriptionGroups) to json";
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public Options buildCommandlineOptions(Options options) {
|
|||
|
|
+ Option pathOption = new Option("p", "path", true,
|
|||
|
|
+ "Absolute path to the metadata directory");
|
|||
|
|
+ pathOption.setRequired(true);
|
|||
|
|
+ options.addOption(pathOption);
|
|||
|
|
+
|
|||
|
|
+ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " +
|
|||
|
|
+ "topics/subscriptionGroups");
|
|||
|
|
+ configTypeOption.setRequired(true);
|
|||
|
|
+ options.addOption(configTypeOption);
|
|||
|
|
+
|
|||
|
|
+ return options;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ @Override
|
|||
|
|
+ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException {
|
|||
|
|
+ String path = commandLine.getOptionValue("path").trim();
|
|||
|
|
+ if (StringUtils.isEmpty(path) || !new File(path).exists()) {
|
|||
|
|
+ System.out.print("Rocksdb path is invalid.\n");
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ String configType = commandLine.getOptionValue("configType").trim().toLowerCase();
|
|||
|
|
+
|
|||
|
|
+ final long memTableFlushInterval = 60 * 60 * 1000L;
|
|||
|
|
+ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval);
|
|||
|
|
+ try {
|
|||
|
|
+ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
+ // for topics.json
|
|||
|
|
+ final Map<String, JSONObject> topicsJsonConfig = new HashMap<>();
|
|||
|
|
+ final Map<String, JSONObject> topicConfigTable = new HashMap<>();
|
|||
|
|
+ boolean isLoad = kvConfigManager.load(path, (key, value) -> {
|
|||
|
|
+ final String topic = new String(key, DataConverter.CHARSET_UTF8);
|
|||
|
|
+ final String topicConfig = new String(value, DataConverter.CHARSET_UTF8);
|
|||
|
|
+ final JSONObject jsonObject = JSONObject.parseObject(topicConfig);
|
|||
|
|
+ topicConfigTable.put(topic, jsonObject);
|
|||
|
|
+ });
|
|||
|
|
+
|
|||
|
|
+ if (isLoad) {
|
|||
|
|
+ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable));
|
|||
|
|
+ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true);
|
|||
|
|
+ System.out.print(topicsJsonStr + "\n");
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) {
|
|||
|
|
+ // for subscriptionGroup.json
|
|||
|
|
+ final Map<String, JSONObject> subscriptionGroupJsonConfig = new HashMap<>();
|
|||
|
|
+ final Map<String, JSONObject> subscriptionGroupTable = new HashMap<>();
|
|||
|
|
+ boolean isLoad = kvConfigManager.load(path, (key, value) -> {
|
|||
|
|
+ final String subscriptionGroup = new String(key, DataConverter.CHARSET_UTF8);
|
|||
|
|
+ final String subscriptionGroupConfig = new String(value, DataConverter.CHARSET_UTF8);
|
|||
|
|
+ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig);
|
|||
|
|
+ subscriptionGroupTable.put(subscriptionGroup, jsonObject);
|
|||
|
|
+ });
|
|||
|
|
+
|
|||
|
|
+ if (isLoad) {
|
|||
|
|
+ subscriptionGroupJsonConfig.put("subscriptionGroupTable",
|
|||
|
|
+ (JSONObject) JSONObject.toJSON(subscriptionGroupTable));
|
|||
|
|
+ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true);
|
|||
|
|
+ System.out.print(subscriptionGroupJsonStr + "\n");
|
|||
|
|
+ return;
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+ System.out.print("Config type was not recognized, configType=" + configType + "\n");
|
|||
|
|
+ } finally {
|
|||
|
|
+ kvConfigManager.stop();
|
|||
|
|
+ }
|
|||
|
|
+ }
|
|||
|
|
+}
|
|||
|
|
--
|
|||
|
|
2.32.0.windows.2
|
|||
|
|
|