From a28174e96285b552c6db62e569a575d12dde287a Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 31 Mar 2026 16:03:38 -0700 Subject: [PATCH 1/3] OM Snapshot UI -- backend code. Change-Id: Iea21514269d37a233c816a2cc55be44fe788382a --- .../webapps/static/templates/menu.html | 2 +- .../ozone/om/OzoneManagerHttpServer.java | 2 + .../ozone/om/SnapshotListJSONServlet.java | 101 ++++++++++++++++++ .../webapps/ozoneManager/om-snapshots.html | 52 +++++++++ .../webapps/ozoneManager/ozoneManager.js | 32 ++++++ .../ozone/om/TestSnapshotListJSONServlet.java | 57 ++++++++++ 6 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotListJSONServlet.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotListJSONServlet.java diff --git a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html index 1963a6543835..d0a6907e586e 100644 --- a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html +++ b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html @@ -58,6 +58,6 @@
  • IO Status
  • Data Scanner
    • -
  • Ozone Snapshot
    • +
  • Snapshots
    • diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java index 9c2688de812d..fd79f153de37 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java @@ -36,6 +36,8 @@ public OzoneManagerHttpServer(MutableConfigurationSource conf, super(conf, "ozoneManager"); addServlet("serviceList", OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT, ServiceListJSONServlet.class); + addServlet("snapshotList", "/snapshotList", + SnapshotListJSONServlet.class); addServlet("dbCheckpoint", OZONE_DB_CHECKPOINT_HTTP_ENDPOINT, OMDBCheckpointServlet.class); addServlet("dbCheckpointv2", OZONE_DB_CHECKPOINT_HTTP_ENDPOINT_V2, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotListJSONServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotListJSONServlet.java new file mode 100644 index 000000000000..97dd26a9e5ce --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotListJSONServlet.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import java.io.IOException; +import java.io.PrintWriter; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides REST access to Ozone Snapshot List. + */ +public class SnapshotListJSONServlet extends HttpServlet { + + private static final Logger LOG = + LoggerFactory.getLogger(SnapshotListJSONServlet.class); + private static final long serialVersionUID = 1L; + + private transient OzoneManager om; + + /** + * Jackson mix-in to ignore protobuf getter in SnapshotInfo. + */ + @JsonIgnoreProperties({"protobuf", "createTransactionInfo", "lastTransactionInfo"}) + abstract static class SnapshotInfoMixin { + } + + @Override + public void init() throws ServletException { + this.om = (OzoneManager) getServletContext() + .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE); + } + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) { + try { + String volume = request.getParameter("volume"); + String bucket = request.getParameter("bucket"); + + if (volume == null || volume.isEmpty() || bucket == null || bucket.isEmpty()) { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + response.getWriter().write("Both volume and bucket parameters are required."); + return; + } + + String prefix = request.getParameter("prefix"); + String startItem = request.getParameter("startItem"); + String maxKeysStr = request.getParameter("maxKeys"); + + int maxKeys = 100; + if (maxKeysStr != null) { + maxKeys = Integer.parseInt(maxKeysStr); + } + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.addMixIn(SnapshotInfo.class, SnapshotInfoMixin.class); + objectMapper.enable(SerializationFeature.INDENT_OUTPUT); + response.setContentType("application/json; charset=utf8"); + PrintWriter writer = response.getWriter(); + try { + ListSnapshotResponse listSnapshotResponse = om.listSnapshot(volume, bucket, prefix, startItem, maxKeys); + writer.write(objectMapper.writeValueAsString(listSnapshotResponse.getSnapshotInfos())); + } finally { + if (writer != null) { + writer.close(); + } + } + } catch (IOException e) { + LOG.error("Caught an exception while processing SnapshotList request", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } catch (Exception e) { + LOG.error("Unexpected error while processing SnapshotList request", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html index ef718260ce05..7f4422dcd22d 100644 --- a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html @@ -14,7 +14,59 @@ See the License for the specific language governing permissions and limitations under the License. --> +

    Snapshots

    +
    +
    +
    + Volume + +
    +
    +
    +
    + Bucket + +
    +
    +
    + +
    +
    + +
    + Please enter both volume and bucket to search snapshots. +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + +
    NameStatusCreation TimeReferenced SizeExclusive SizeSnapshot IDSnapshot Path
    {{snapshot.name}} + {{snapshot.snapshotStatus === 'SNAPSHOT_ACTIVE' ? 'active' : 'deletion in progress'}} + {{snapshot.creationTime | date:'yyyy-MM-dd HH:mm:ss'}}{{$ctrl.formatBytes(snapshot.referencedSize)}}{{$ctrl.formatBytes(snapshot.exclusiveSize)}}{{snapshot.snapshotId}}{{snapshot.snapshotPath}}
    +

    Snapshot Diff Jobs

    diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js index e98d3f7ba3a7..8ad270e8304a 100644 --- a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js @@ -38,11 +38,38 @@ var ctrl = this; ctrl.snapshotMetrics = []; ctrl.snapshotDiffJobs = []; + ctrl.snapshots = []; ctrl.snapshotUsageMetrics = { 'NumSnapshotActive': 0, 'NumSnapshotDeleted': 0, 'NumSnapshotCacheSize': 0 }; + + ctrl.listSnapshots = function(volume, bucket) { + if (volume && bucket) { + $http.get("snapshotList?volume=" + volume + "&bucket=" + bucket) + .then(function (result) { + ctrl.snapshots = result.data; + }) + .catch(function (error) { + console.error("Error fetching snapshots:", error); + ctrl.snapshots = []; + }); + } else { + ctrl.snapshots = []; + } + }; + + ctrl.formatBytes = function(bytes, decimals) { + if (bytes == 0) return '0 Bytes'; + if (!bytes) return 'N/A'; + var k = 1024, + dm = decimals + 1 || 3, + sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'], + i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; + } + $scope.reverse = false; $scope.columnName = "jobId"; let snapDiffJobsCopy = []; @@ -57,6 +84,11 @@ ctrl.snapshotUsageMetrics.NumSnapshotActive = metrics.NumSnapshotActive || 0; ctrl.snapshotUsageMetrics.NumSnapshotDeleted = metrics.NumSnapshotDeleted || 0; ctrl.snapshotUsageMetrics.NumSnapshotCacheSize = metrics.NumSnapshotCacheSize || 0; + for (var key in metrics) { + if (key.match(/NumSnapshot|NumCancelSnapshotDiff|NumListSnapshotDiffJob/)) { + ctrl.snapshotMetrics.push({key: key, value: metrics[key]}); + } + } } }); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotListJSONServlet.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotListJSONServlet.java new file mode 100644 index 000000000000..a3bab726ccf4 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotListJSONServlet.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Collections; +import java.util.UUID; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.util.Time; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test for SnapshotListJSONServlet. + */ +public class TestSnapshotListJSONServlet { + + @Test + public void testSerialization() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.addMixIn(SnapshotInfo.class, + SnapshotListJSONServlet.SnapshotInfoMixin.class); + + String snapName = "snap1"; + SnapshotInfo snapshotInfo = SnapshotInfo.newInstance("vol1", "bucket1", + snapName, UUID.randomUUID(), Time.now()); + + String json = assertDoesNotThrow(() -> + objectMapper.writeValueAsString(Collections.singletonList(snapshotInfo))); + + // Verify that the problematic fields are NOT in the JSON + assertFalse(json.contains("protobuf"), "JSON should not contain protobuf field"); + assertFalse(json.contains("createTransactionInfo"), "JSON should not contain createTransactionInfo field"); + assertFalse(json.contains("lastTransactionInfo"), "JSON should not contain lastTransactionInfo field"); + + // Verify that the expected fields ARE in the JSON + assertTrue(json.contains("\"name\":\"" + snapName + "\""), "JSON should contain snapshot name"); + } +} From d09f5f1eb51f79548b159da9b46debc0f49ae5f4 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 13 Apr 2026 16:24:56 -0700 Subject: [PATCH 2/3] HDDS-14960. Fix checkstyle issues in TestSnapshotListJSONServlet Change-Id: I55dd9b8a4fa31bcacefd83fbb827627c2d006fda --- .../hadoop/ozone/om/TestSnapshotListJSONServlet.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotListJSONServlet.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotListJSONServlet.java index a3bab726ccf4..7458feb9d5b1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotListJSONServlet.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotListJSONServlet.java @@ -17,6 +17,10 @@ package org.apache.hadoop.ozone.om; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Collections; import java.util.UUID; @@ -24,10 +28,6 @@ import org.apache.hadoop.util.Time; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Test for SnapshotListJSONServlet. */ From 72a1b13faa46bc7478a05d0dfc0cf8c2d94e80e4 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 20 Apr 2026 23:04:34 -0700 Subject: [PATCH 3/3] Fix Change-Id: I6fb1ee4df80664310c3e91d0f4aa51f077eea545 --- .../ozone/om/SnapshotListJSONServlet.java | 23 ++++++++----------- .../webapps/ozoneManager/om-snapshots.html | 2 +- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotListJSONServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotListJSONServlet.java index 97dd26a9e5ce..5f0091be3132 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotListJSONServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotListJSONServlet.java @@ -26,6 +26,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; @@ -69,26 +70,20 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { } String prefix = request.getParameter("prefix"); - String startItem = request.getParameter("startItem"); - String maxKeysStr = request.getParameter("maxKeys"); - int maxKeys = 100; - if (maxKeysStr != null) { - maxKeys = Integer.parseInt(maxKeysStr); - } + final int maxKeys = 100; ObjectMapper objectMapper = new ObjectMapper(); objectMapper.addMixIn(SnapshotInfo.class, SnapshotInfoMixin.class); objectMapper.enable(SerializationFeature.INDENT_OUTPUT); response.setContentType("application/json; charset=utf8"); - PrintWriter writer = response.getWriter(); - try { - ListSnapshotResponse listSnapshotResponse = om.listSnapshot(volume, bucket, prefix, startItem, maxKeys); - writer.write(objectMapper.writeValueAsString(listSnapshotResponse.getSnapshotInfos())); - } finally { - if (writer != null) { - writer.close(); - } + try (PrintWriter writer = response.getWriter()) { + String lastSnapshot = null; + do { + ListSnapshotResponse listSnapshotResponse = om.listSnapshot(volume, bucket, prefix, lastSnapshot, maxKeys); + writer.write(objectMapper.writeValueAsString(listSnapshotResponse.getSnapshotInfos())); + lastSnapshot = listSnapshotResponse.getLastSnapshot(); + } while (StringUtils.isNotEmpty(lastSnapshot)); } } catch (IOException e) { LOG.error("Caught an exception while processing SnapshotList request", e); diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html index 7f4422dcd22d..eafb3f179416 100644 --- a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html @@ -59,7 +59,7 @@

    Snapshots

    {{snapshot.creationTime | date:'yyyy-MM-dd HH:mm:ss'}} {{$ctrl.formatBytes(snapshot.referencedSize)}} - {{$ctrl.formatBytes(snapshot.exclusiveSize)}} + {{$ctrl.formatBytes(snapshot.exclusiveSize + snapshot.exclusiveSizeDeltaFromDirDeepCleaning)}} {{snapshot.snapshotId}} {{snapshot.snapshotPath}}