diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 2c44fa881c41..4bb4f32d57c8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -628,6 +628,10 @@ public final class ScmConfigKeys { public static final String OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT = "ozone.scm.ha.raft.server.rpc.first-election.timeout"; + public static final String OZONE_SCM_RATIS_EVENTS_MAX_LIMIT = + "ozone.scm.ratis.events.max.limit"; + public static final int OZONE_SCM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100; + /** * Never constructed. */ diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index e159eb6948b8..45207cb96fea 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -5044,4 +5044,16 @@ 5m Interval for cleaning up orphan snapshot local data versions corresponding to snapshots + + ozone.scm.ratis.events.max.limit + 100 + OZONE, RATIS, SCM + The maximum number of events that can be pending in SCM Ratis. + + + ozone.om.ratis.events.max.limit + 100 + OZONE, RATIS, OM + The maximum number of events that can be pending in OM Ratis. + diff --git a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html index 1963a6543835..35a3410ef032 100644 --- a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html +++ b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html @@ -34,6 +34,7 @@
  • Configuration
    • +
  • Ratis event timeline
  • Documentation
  • ratisEvents = new LinkedList<>(); + private final int maxRatisEvents; + /** * Container stat metrics, the meaning of following metrics * can be found in {@link ContainerStat}. @@ -59,14 +66,23 @@ public DBCheckpointMetrics getDBCheckpointMetrics() { return dbCheckpointMetrics; } - public SCMMetrics() { + public SCMMetrics(int maxRatisEvents) { dbCheckpointMetrics = DBCheckpointMetrics.create("SCM Metrics"); + this.maxRatisEvents = maxRatisEvents; } public static SCMMetrics create() { + return create(null); + } + + public static SCMMetrics create(ConfigurationSource conf) { MetricsSystem ms = DefaultMetricsSystem.instance(); + int maxRatisEvents = conf == null + ? ScmConfigKeys.OZONE_SCM_RATIS_EVENTS_MAX_LIMIT_DEFAULT + : conf.getInt(ScmConfigKeys.OZONE_SCM_RATIS_EVENTS_MAX_LIMIT, + ScmConfigKeys.OZONE_SCM_RATIS_EVENTS_MAX_LIMIT_DEFAULT); return ms.register(SOURCE_NAME, "Storage Container Manager Metrics", - new SCMMetrics()); + new SCMMetrics(maxRatisEvents)); } public void setLastContainerReportSize(long size) { @@ -155,6 +171,22 @@ public void decrContainerStat(ContainerStat deltaStat) { this.containerReportWriteCount.incr(-1 * deltaStat.getWriteCount().get()); } + public void addRatisEvent(String event) { + synchronized (ratisEvents) { + if (ratisEvents.size() >= maxRatisEvents) { + ratisEvents.removeFirst(); + } + ratisEvents.add(Time.formatTime(Time.now()) + "|" + event); + } + } + + @Metric("Ratis state machine events") + public String getRatisEvents() { + synchronized (ratisEvents) { + return String.join("\n", ratisEvents); + } + } + public void unRegister() { MetricsSystem ms = DefaultMetricsSystem.instance(); ms.unregisterSource(SOURCE_NAME); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java index 9d49ca36b6f2..59b56b6dba4a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType; import org.apache.hadoop.hdds.scm.block.DeletedBlockLog; import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; @@ -78,6 +79,7 @@ public class SCMStateMachine extends BaseStateMachine { private StorageContainerManager scm; private Map handlers; private SCMHADBTransactionBuffer transactionBuffer; + private final SCMMetrics metrics; private final SimpleStateMachineStorage storage = new SimpleStateMachineStorage(); private final boolean isInitialized; @@ -94,6 +96,7 @@ public SCMStateMachine(final StorageContainerManager scm, this.scm = scm; this.handlers = new EnumMap<>(RequestType.class); this.transactionBuffer = buffer; + this.metrics = scm.getMetrics(); TransactionInfo latestTrxInfo = this.transactionBuffer.getLatestTrxInfo(); if (!latestTrxInfo.isDefault()) { updateLastAppliedTermIndex(latestTrxInfo.getTerm(), @@ -111,12 +114,19 @@ public SCMStateMachine(final StorageContainerManager scm, public SCMStateMachine() { isInitialized = false; + this.metrics = null; } public void registerHandler(RequestType type, Object handler) { handlers.put(type, handler); } + private void addRatisEvent(String message) { + if (metrics != null) { + metrics.addRatisEvent(message); + } + } + @Override public SnapshotInfo getLatestSnapshot() { // Transaction buffer will be null during scm initlialization phase @@ -212,7 +222,9 @@ public void notifyNotLeader(Collection pendingEntries) { if (!isInitialized) { return; } - LOG.info("current leader SCM steps down."); + String message = "SCM " + scm.getScmId() + " steps down from being leader."; + LOG.info(message); + addRatisEvent(message); scm.getScmContext().updateLeaderAndTerm(false, 0); scm.getSCMServiceManager().notifyStatusChanged(); @@ -243,6 +255,8 @@ public CompletableFuture notifyInstallSnapshotFromLeader( final String leaderNodeId = leaderDetails.get().getNodeId(); LOG.info("Received install snapshot notification from SCM leader: {} with " + "term index: {}", leaderAddress, firstTermIndexInLog); + addRatisEvent("Installing snapshot from SCM leader " + leaderNodeId + + ", term index: " + firstTermIndexInLog); CompletableFuture future = CompletableFuture.supplyAsync( () -> { @@ -296,11 +310,17 @@ public void notifyLeaderChanged(RaftGroupMemberId groupMemberId, } if (!groupMemberId.getPeerId().equals(newLeaderId)) { - LOG.info("leader changed, yet current SCM is still follower."); + String message = "Leader changed to " + newLeaderId + + ", current SCM " + scm.getScmId() + " is still follower."; + LOG.info(message); + addRatisEvent(message); return; } - LOG.info("current SCM becomes leader of term {}.", currentLeaderTerm); + String message = "current SCM " + scm.getScmId() + + " becomes leader of term " + currentLeaderTerm; + LOG.info(message); + addRatisEvent(message); scm.getScmContext().updateLeaderAndTerm(true, currentLeaderTerm.get()); @@ -394,11 +414,15 @@ public void notifyLeaderReady() { scm.getScmContext().setLeaderReady(); scm.getSCMServiceManager().notifyStatusChanged(); scm.getFinalizationManager().onLeaderReady(); + addRatisEvent("SCM " + scm.getScmId() + + " is ready to serve requests as the leader"); } @Override public void notifyConfigurationChanged(long term, long index, RaftProtos.RaftConfigurationProto newRaftConfiguration) { + addRatisEvent("Configuration changed at term index (" + term + ", " + index + + ") to " + newRaftConfiguration.toString()); } @Override @@ -431,6 +455,7 @@ public void reinitialize() throws IOException { } LOG.info("{}: SCMStateMachine is reinitializing. newTermIndex = {}", getId(), termIndex); + addRatisEvent("reinitialize: " + termIndex); // re-initialize the DBTransactionBuffer and update the lastAppliedIndex. try { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 1fc7c2d96b24..e563c9f08ffc 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -25,7 +25,6 @@ import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.IO_EXCEPTION; import static org.apache.hadoop.hdds.scm.net.NetConstants.NODE_COST_DEFAULT; import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT; -import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.getPerfMetrics; import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer; import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress; @@ -108,7 +107,7 @@ public class SCMBlockProtocolServer implements public SCMBlockProtocolServer(OzoneConfiguration conf, StorageContainerManager scm) throws IOException { this.scm = scm; - this.perfMetrics = getPerfMetrics(); + this.perfMetrics = scm.getPerfMetrics(); final int handlerCount = conf.getInt(OZONE_SCM_BLOCK_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT, LOG::info); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 696816b85a24..69f7973ff1bd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -226,8 +226,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl /** * SCM metrics. */ - private static SCMMetrics metrics; - private static SCMPerformanceMetrics perfMetrics; + private SCMMetrics metrics; + private SCMPerformanceMetrics perfMetrics; private SCMHAMetrics scmHAMetrics; private final NettyMetrics nettyMetrics; @@ -1427,28 +1427,28 @@ private static InetSocketAddress getScmAddress(SCMHANodeDetails haDetails, /** * Initialize SCM metrics. */ - public static void initMetrics() { - metrics = SCMMetrics.create(); + public void initMetrics() { + metrics = SCMMetrics.create(configuration); } /** * Return SCM metrics instance. */ - public static SCMMetrics getMetrics() { - return metrics == null ? SCMMetrics.create() : metrics; + public SCMMetrics getMetrics() { + return metrics; } /** * Initialize SCMPerformance metrics. */ - public static void initPerfMetrics() { + public void initPerfMetrics() { perfMetrics = SCMPerformanceMetrics.create(); } /** * Return SCMPerformance metrics instance. */ - public static SCMPerformanceMetrics getPerfMetrics() { + public SCMPerformanceMetrics getPerfMetrics() { return perfMetrics == null ? SCMPerformanceMetrics.create() : perfMetrics; } diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/ratis-events.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/ratis-events.html new file mode 100644 index 000000000000..9eb6008d00ff --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/ratis-events.html @@ -0,0 +1,32 @@ + +

    Ratis event timeline

    + + + + + + + + + + + + + + +
    TimestampEvent description
    {{event.timestamp}}{{event.description}}
    diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js index eca79852e43b..1e44826f980e 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js @@ -19,6 +19,32 @@ "use strict"; angular.module('scm', ['ozone', 'nvd3']); + angular.module('scm').config(function ($routeProvider) { + $routeProvider + .when("/ratis_events", { + template: "" + }); + }); + + angular.module('scm').component('ratisEvents', { + templateUrl: 'ratis-events.html', + controller: function ($http) { + var ctrl = this; + $http.get("jmx?qry=Hadoop:service=StorageContainerManager,name=SCMMetrics") + .then(function (result) { + var metrics = result.data.beans[0]; + var rawEvents = metrics['tag.RatisEvents'] ? metrics['tag.RatisEvents'].split('\n') : []; + ctrl.events = rawEvents.map(function(e) { + var parts = e.split('|'); + return { + timestamp: parts[0], + description: parts[1] + }; + }); + }); + } + }); + angular.module('scm').component('scmOverview', { templateUrl: 'scm-overview.html', require: { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java new file mode 100644 index 000000000000..828606f2d424 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.ha; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.server.protocol.TermIndex; +import org.junit.jupiter.api.Test; + +/** + * Test SCMStateMachine events recording. + */ +public class TestSCMStateMachine { + + @Test + public void testRatisEventsRecording() throws Exception { + StorageContainerManager scm = mock(StorageContainerManager.class); + SCMMetrics metrics = SCMMetrics.create(); + when(scm.getMetrics()).thenReturn(metrics); + + SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class); + when(buffer.getLatestTrxInfo()).thenReturn(TransactionInfo.valueOf(TermIndex.valueOf(0, 0))); + + SCMStateMachine stateMachine = new SCMStateMachine(scm, buffer); + + stateMachine.notifyConfigurationChanged(1, 1, RaftProtos.RaftConfigurationProto.getDefaultInstance()); + assertTrue(metrics.getRatisEvents().contains("Configuration changed at term index")); + + metrics.unRegister(); + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 05083d506304..6d83209dd0fd 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -691,6 +691,10 @@ public final class OMConfigKeys { "ozone.om.snapshot.local.data.manager.service.interval"; public static final String OZONE_OM_SNAPSHOT_LOCAL_DATA_MANAGER_SERVICE_INTERVAL_DEFAULT = "5m"; + public static final String OZONE_OM_RATIS_EVENTS_MAX_LIMIT = + "ozone.om.ratis.events.max.limit"; + public static final int OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100; + /** * Never constructed. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java index cb8045e0e6e5..7054e350682f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java @@ -87,7 +87,7 @@ public void init() throws Exception { .build(); cluster.waitForClusterToBeReady(); StorageContainerManager scm = cluster.getStorageContainerManager(); - scmMetrics = StorageContainerManager.getMetrics(); + scmMetrics = scm.getMetrics(); requestMock = mock(HttpServletRequest.class); when(requestMock.getParameter(OZONE_DB_CHECKPOINT_REQUEST_FLUSH)) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRangerBGSyncService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRangerBGSyncService.java index 9f5690e11e4c..a01bcf370484 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRangerBGSyncService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRangerBGSyncService.java @@ -177,7 +177,7 @@ public void setUp() throws IOException { Path metaDirPath = Paths.get(path, "om-meta"); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString()); - omMetrics = OMMetrics.create(); + omMetrics = OMMetrics.create(conf); conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.resolve("om").toAbsolutePath().toString()); // No need to conf.set(OzoneConfigKeys.OZONE_ADMINISTRATORS, ...) here diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java index 9d2cab2fc507..5a70483b2bc8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java @@ -18,7 +18,9 @@ package org.apache.hadoop.ozone.om; import com.google.common.annotations.VisibleForTesting; +import java.util.LinkedList; import org.apache.hadoop.hdds.annotation.InterfaceAudience; +import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.DBCheckpointMetrics; import org.apache.hadoop.metrics2.MetricsSystem; @@ -28,6 +30,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.ozone.om.snapshot.OMSnapshotDirectoryMetrics; +import org.apache.hadoop.util.Time; /** * This class is for maintaining Ozone Manager statistics. @@ -38,6 +41,9 @@ public class OMMetrics implements OmMetadataReaderMetrics { private static final String SOURCE_NAME = OMMetrics.class.getSimpleName(); + private final LinkedList ratisEvents = new LinkedList<>(); + private final int maxRatisEvents; + // OM request type op metrics private @Metric MutableCounterLong numVolumeOps; private @Metric MutableCounterLong numBucketOps; @@ -256,15 +262,20 @@ public class OMMetrics implements OmMetadataReaderMetrics { private final DBCheckpointMetrics dbCheckpointMetrics; private OMSnapshotDirectoryMetrics snapshotDirectoryMetrics; - public OMMetrics() { + public OMMetrics(int maxRatisEvents) { dbCheckpointMetrics = DBCheckpointMetrics.create("OM Metrics"); + this.maxRatisEvents = maxRatisEvents; } - public static OMMetrics create() { + public static OMMetrics create(ConfigurationSource conf) { MetricsSystem ms = DefaultMetricsSystem.instance(); + int maxRatisEvents = conf == null + ? OMConfigKeys.OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT + : conf.getInt(OMConfigKeys.OZONE_OM_RATIS_EVENTS_MAX_LIMIT, + OMConfigKeys.OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT); return ms.register(SOURCE_NAME, "Ozone Manager Metrics", - new OMMetrics()); + new OMMetrics(maxRatisEvents)); } public DBCheckpointMetrics getDBCheckpointMetrics() { @@ -1570,6 +1581,22 @@ public void incNumRecoverLeaseFails() { numRecoverLeaseFails.incr(); } + public void addRatisEvent(String event) { + synchronized (ratisEvents) { + if (ratisEvents.size() >= maxRatisEvents) { + ratisEvents.removeFirst(); + } + ratisEvents.add(Time.formatTime(Time.now()) + "|" + event); + } + } + + @Metric("Ratis state machine events") + public String getRatisEvents() { + synchronized (ratisEvents) { + return String.join("\n", ratisEvents); + } + } + public void unRegister() { if (dbCheckpointMetrics != null) { dbCheckpointMetrics.unRegister(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index e3ec61c5b469..0bd52f68d875 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -693,7 +693,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) this.isS3MultiTenancyEnabled = OMMultiTenantManager.checkAndEnableMultiTenancy(this, conf); - metrics = OMMetrics.create(); + metrics = OMMetrics.create(conf); omSnapshotIntMetrics = OmSnapshotInternalMetrics.create(); perfMetrics = OMPerformanceMetrics.register(); omDeletionMetrics = DeletingServiceMetrics.create(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java index 09a530ab6cc7..2abaf9ae5719 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java @@ -26,6 +26,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -41,6 +42,7 @@ import org.apache.hadoop.ozone.audit.AuditLoggerType; import org.apache.hadoop.ozone.audit.OMSystemAction; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.OzoneManagerPrepareState; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -170,6 +172,10 @@ public synchronized void reinitialize() throws IOException { final TermIndex lastApplied = getLastAppliedTermIndex(); unpause(lastApplied.getIndex(), lastApplied.getTerm()); LOG.info("{}: reinitialize {} with {}", getId(), getGroupId(), lastApplied); + OMMetrics metrics = ozoneManager.getMetrics(); + if (metrics != null) { + metrics.addRatisEvent("reinitialize: " + lastApplied); + } } } @@ -183,6 +189,19 @@ public SnapshotInfo getLatestSnapshot() { @Override public void notifyLeaderReady() { ozoneManager.getOmSnapshotManager().resetInFlightSnapshotCount(); + OMMetrics metrics = ozoneManager.getMetrics(); + if (metrics != null) { + metrics.addRatisEvent("Ready to serve requests as the leader"); + } + } + + @Override + public void notifyNotLeader(Collection pendingEntries) { + LOG.info("current leader OM steps down."); + OMMetrics metrics = ozoneManager.getMetrics(); + if (metrics != null) { + metrics.addRatisEvent("current leader OM steps down."); + } } @Override @@ -208,6 +227,10 @@ public void notifyLeaderChanged(RaftGroupMemberId groupMemberId, AUDIT.logWriteSuccess(ozoneManager.buildAuditMessageForSuccess(OMSystemAction.LEADER_CHANGE, auditParams)); LOG.info("{}: leader changed to {}", groupMemberId, newLeaderId); + OMMetrics metrics = ozoneManager.getMetrics(); + if (metrics != null) { + metrics.addRatisEvent("Leader changed to " + newLeaderId); + } } /** Notified by Ratis for non-StateMachine term-index update. */ @@ -282,10 +305,22 @@ public void notifyConfigurationChanged(long term, long index, for (RaftProtos.RaftPeerProto raftPeerProto : newPeers) { newPeerIds.add(RaftPeerId.valueOf(raftPeerProto.getId()).toString()); } - for (RaftProtos.RaftPeerProto raftPeerProto : newListeners) { - newPeerIds.add(RaftPeerId.valueOf(raftPeerProto.getId()).toString()); + List newListenersIds = new ArrayList<>(); + for (RaftProtos.RaftPeerProto raftListenerProto : newListeners) { + newListenersIds.add(RaftPeerId.valueOf(raftListenerProto.getId()).toString()); + } + + OMMetrics metrics = ozoneManager.getMetrics(); + if (metrics != null) { + metrics.addRatisEvent( + "New peers " + newPeerIds + + (newListenersIds.isEmpty() ? "" : ", new listeners " + newListenersIds) + + " added at term index (" + + term + ", " + index + ")"); } + // Check and update the peer list in OzoneManager + newPeerIds.addAll(newListenersIds); ozoneManager.updatePeerList(newPeerIds); } @@ -301,6 +336,11 @@ public void notifySnapshotInstalled(RaftProtos.InstallSnapshotResult result, long snapshotIndex, RaftPeer peer) { LOG.info("Receive notifySnapshotInstalled event {} for the peer: {}" + " snapshotIndex: {}.", result, peer.getId(), snapshotIndex); + OMMetrics metrics = ozoneManager.getMetrics(); + if (metrics != null) { + metrics.addRatisEvent("Install snapshot " + + result + ", snapshotIndex=" + snapshotIndex + ", peer=" + peer.getId()); + } switch (result) { case SUCCESS: case SNAPSHOT_UNAVAILABLE: @@ -581,6 +621,11 @@ public CompletableFuture notifyInstallSnapshotFromLeader( .getLeaderInfo().getId().getId()).toString(); LOG.info("Received install snapshot notification from OM leader: {} with " + "term index: {}", leaderNodeId, firstTermIndexInLog); + OMMetrics metrics = ozoneManager.getMetrics(); + if (metrics != null) { + metrics.addRatisEvent("Installing snapshot from " + + "OM leader " + leaderNodeId + ", term index: " + firstTermIndexInLog); + } return CompletableFuture.supplyAsync( () -> { diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js index e98d3f7ba3a7..8b64a082362f 100644 --- a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js @@ -30,6 +30,9 @@ }) .when("/snapshots", { template: "" + }) + .when("/ratis_events", { + template: "" }); }); angular.module('ozoneManager').component('omSnapshots', { @@ -131,6 +134,24 @@ } } }); + angular.module('ozoneManager').component('ratisEvents', { + templateUrl: 'ratis-events.html', + controller: function ($http) { + var ctrl = this; + $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics") + .then(function (result) { + var metrics = result.data.beans[0]; + var rawEvents = metrics['tag.RatisEvents'] ? metrics['tag.RatisEvents'].split('\n') : []; + ctrl.events = rawEvents.map(function(e) { + var parts = e.split('|'); + return { + timestamp: parts[0], + description: parts[1] + }; + }); + }); + } + }); angular.module('ozoneManager').component('omMetrics', { templateUrl: 'om-metrics.html', controller: function ($http) { diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ratis-events.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ratis-events.html new file mode 100644 index 000000000000..9eb6008d00ff --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ratis-events.html @@ -0,0 +1,32 @@ + +

    Ratis event timeline

    + + + + + + + + + + + + + + +
    TimestampEvent description
    {{event.timestamp}}{{event.description}}
    diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java index 81a0691a749a..c0ad927bc059 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java @@ -106,8 +106,8 @@ class TestOzoneManagerDoubleBuffer { @BeforeEach public void setup() throws IOException { - OMMetrics omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + OMMetrics omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, tempDir.getAbsolutePath()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java index 4ba3c6b5d1b7..8b3027d12cd1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java @@ -91,8 +91,8 @@ public class TestOzoneManagerDoubleBufferWithOMResponse { @BeforeEach public void setup() throws IOException { ozoneManager = mock(OzoneManager.class, withSettings().stubOnly()); - OMMetrics omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + OMMetrics omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java index 36d7a80aeea9..111779b95734 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java @@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.audit.OMSystemAction; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmConfig; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshotManager; @@ -1037,4 +1038,28 @@ private RaftClientRequest buildClientRequest( .setType(RaftClientRequest.writeRequestType()) .build(); } + + @Test + public void testRatisEventsRecording() { + OzoneConfiguration conf = new OzoneConfiguration(); + OMMetrics metrics = OMMetrics.create(conf); + when(om.getMetrics()).thenReturn(metrics); + when(om.getOmSnapshotManager()).thenReturn(mock(OmSnapshotManager.class)); + when(om.getConfiguration()).thenReturn(conf); + AuditMessage auditMessage = mock(AuditMessage.class); + when(auditMessage.getOp()).thenReturn("LEADER_CHANGE"); + when(om.buildAuditMessageForSuccess(any(), any())).thenReturn(auditMessage); + sm.notifyLeaderReady(); + assertTrue(metrics.getRatisEvents().contains("Ready to serve requests as the leader")); + + RaftGroupMemberId groupMemberId = mock(RaftGroupMemberId.class); + when(groupMemberId.getPeerId()).thenReturn(RaftPeerId.valueOf("peer1")); + sm.notifyLeaderChanged(groupMemberId, RaftPeerId.valueOf("peer1")); + assertTrue(metrics.getRatisEvents().contains("Leader changed to peer1")); + + sm.notifyConfigurationChanged(1, 1, RaftProtos.RaftConfigurationProto.getDefaultInstance()); + assertTrue(metrics.getRatisEvents().contains("New peers [] added at term index ")); + + metrics.unRegister(); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java index 145ffe9854fe..38036912b96b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java @@ -70,8 +70,8 @@ public class TestOMClientRequestWithUserInfo { @BeforeEach public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); - OMMetrics omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + OMMetrics omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); OMMetadataManager omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java index 018e60633a50..40f54be4cc03 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java @@ -59,8 +59,8 @@ public class TestBucketRequest { public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java index 1b096070ea3f..7adf92518d18 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java @@ -92,8 +92,8 @@ public class TestOMDirectoryCreateRequest { @BeforeEach public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java index 164ccf661cce..8f5b6c807caa 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java @@ -92,8 +92,8 @@ public class TestOMDirectoryCreateRequestWithFSO { @BeforeEach public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); OMRequestTestUtils.configureFSOptimizedPaths(ozoneConfiguration, true); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index b84294370c58..405cc706ef90 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -136,10 +136,10 @@ public class TestOMKeyRequest { @BeforeEach public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); + OzoneConfiguration ozoneConfiguration = getOzoneConfiguration(); + omMetrics = OMMetrics.create(ozoneConfiguration); perfMetrics = OMPerformanceMetrics.register(); delMetrics = DeletingServiceMetrics.create(); - OzoneConfiguration ozoneConfiguration = getOzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java index 38749f6812a6..15ed924cc408 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java @@ -77,8 +77,8 @@ public class TestS3MultipartRequest { @BeforeEach public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3GetSecretRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3GetSecretRequest.java index 78ccef961f38..0067b2b38047 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3GetSecretRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3GetSecretRequest.java @@ -133,8 +133,8 @@ public void setUp() throws Exception { when(call.getRemoteUser()).thenReturn(ugiAlice); Server.getCurCall().set(call); - omMetrics = OMMetrics.create(); OzoneConfiguration conf = new OzoneConfiguration(); + omMetrics = OMMetrics.create(conf); conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); // No need to conf.set(OzoneConfigKeys.OZONE_ADMINISTRATORS, ...) here diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantCreateRequest.java index 36529d01064c..4f1ff1659c15 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantCreateRequest.java @@ -72,8 +72,8 @@ public class TestOMTenantCreateRequest { @BeforeEach public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantDeleteRequest.java index 6de4f1d8446d..8accc44fd9d6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantDeleteRequest.java @@ -69,8 +69,8 @@ public class TestOMTenantDeleteRequest { @BeforeEach public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeRequest.java index 36c6034207c0..71cb1b166277 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeRequest.java @@ -61,8 +61,8 @@ public class TestOMVolumeRequest { @BeforeEach public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + omMetrics = OMMetrics.create(ozoneConfiguration); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java index ebc78b26c1e8..96fdb35d4679 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java @@ -82,6 +82,7 @@ class TestSnapshotCache { private static IOzoneManagerLock lock; private SnapshotCache snapshotCache; + private OzoneConfiguration conf; private OMMetrics omMetrics; @BeforeAll @@ -94,8 +95,9 @@ static void beforeAll() throws Exception { @BeforeEach void setUp() throws Exception { + conf = new OzoneConfiguration(); // Reset cache for each test case - omMetrics = OMMetrics.create(); + omMetrics = OMMetrics.create(conf); // Create a difference mock OmSnapshot instance each time load() is called doAnswer((Answer) invocation -> { final OmSnapshot omSnapshot = mock(OmSnapshot.class); @@ -197,7 +199,7 @@ public void testLockHoldsWriteLock(int numberOfLocks) { @Test public void testLockSupplierReturnsLockWithAnotherLockReleased() { - IOzoneManagerLock ozoneManagerLock = new OzoneManagerLock(new OzoneConfiguration()); + IOzoneManagerLock ozoneManagerLock = new OzoneManagerLock(conf); snapshotCache = new SnapshotCache(cacheLoader, CACHE_SIZE_LIMIT, omMetrics, 50, true, ozoneManagerLock); try (UncheckedAutoCloseableSupplier lockDetails = snapshotCache.lock()) { ozoneManagerLock.acquireWriteLock(VOLUME_LOCK, "vol1"); @@ -460,7 +462,7 @@ void testEviction3WithClose() throws IOException, InterruptedException, TimeoutE @Test @DisplayName("Snapshot operations not blocked during compaction") void testSnapshotOperationsNotBlockedDuringCompaction() throws IOException, InterruptedException, TimeoutException { - omMetrics = OMMetrics.create(); + omMetrics = OMMetrics.create(conf); snapshotCache = new SnapshotCache(cacheLoader, 1, omMetrics, 50, true, lock); final UUID dbKey1 = UUID.randomUUID(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 4da22a90ed85..9b573a574229 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -188,7 +188,6 @@ public class TestSnapshotDiffManager { private final List snapshotNames = new ArrayList<>(); private final List snapshotInfoList = new ArrayList<>(); private final List snapDiffJobs = new ArrayList<>(); - private final OMMetrics omMetrics = OMMetrics.create(); @TempDir private File dbDir; @Mock @@ -229,6 +228,9 @@ public void init() throws RocksDBException, IOException, ExecutionException { ExitUtils.disableSystemExit(); ExitUtil.disableSystemExit(); + OzoneConfiguration conf = new OzoneConfiguration(); + OMMetrics omMetrics = OMMetrics.create(conf); + dbOptions = new ManagedDBOptions(); dbOptions.setCreateIfMissing(true); columnFamilyOptions = new ManagedColumnFamilyOptions(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java index 9c6f033b907b..e3776a5b8372 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java @@ -142,9 +142,9 @@ protected TestSnapshotRequestAndResponse(boolean isAdmin) { @BeforeEach public void baseSetup() throws Exception { ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); - omSnapshotIntMetrics = OmSnapshotInternalMetrics.create(); OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + omMetrics = OMMetrics.create(ozoneConfiguration); + omSnapshotIntMetrics = OmSnapshotInternalMetrics.create(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, testDir.getAbsolutePath()); ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS,