From 7eb4f23cb9e0e77043bf78ff714877c11abace76 Mon Sep 17 00:00:00 2001 From: Prathmesh Sapate Date: Mon, 22 Jun 2026 14:18:53 +0530 Subject: [PATCH 1/2] HDDS-15333. Expose volume and bucket utilization metrics on OM /prom Added VolumeUtilizationMetrics with volume used/available bytes, register them on OM, and expose bucket used namespace. --- .../hadoop/ozone/om/OMMetadataManager.java | 3 + .../ozone/om/BucketUtilizationMetrics.java | 7 +- .../ozone/om/OmMetadataManagerImpl.java | 6 + .../apache/hadoop/ozone/om/OzoneManager.java | 6 + .../ozone/om/VolumeUtilizationMetrics.java | 135 ++++++++++++++++ .../om/TestBucketUtilizationMetrics.java | 12 +- .../om/TestVolumeUtilizationMetrics.java | 153 ++++++++++++++++++ 7 files changed, 317 insertions(+), 5 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeUtilizationMetrics.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVolumeUtilizationMetrics.java diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index be66ffc195b5..3dd7007765b9 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -561,6 +561,9 @@ List getMultipartUploadKeys(String volumeName, Iterator, CacheValue>> getBucketIterator(); + Iterator, CacheValue>> + getVolumeIterator(); + TableIterator> getKeyIterator() throws IOException; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketUtilizationMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketUtilizationMetrics.java index 270db6357015..b0c0b440edf2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketUtilizationMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketUtilizationMetrics.java @@ -37,10 +37,11 @@ * Available metrics: *
    *
  • Bytes used in bucket. - *
  • Bucket quote in bytes. + *
  • Bucket quota in bytes. *
  • Bucket quota in namespace. + *
  • Bucket used namespace (number of keys/directories in bucket). *
  • Bucket available space. Calculated from difference between used bytes in bucket and bucket quota. - * If bucket quote is not set then this metric show -1 as value. + * If bucket quota is not set then this metric shows -1 as value. *
*/ @InterfaceAudience.Private @@ -87,6 +88,7 @@ public void getMetrics(MetricsCollector collector, boolean all) { .addGauge(BucketMetricsInfo.BucketSnapshotUsedBytes, bucketInfo.getSnapshotUsedBytes()) .addGauge(BucketMetricsInfo.BucketQuotaBytes, bucketInfo.getQuotaInBytes()) .addGauge(BucketMetricsInfo.BucketQuotaNamespace, bucketInfo.getQuotaInNamespace()) + .addGauge(BucketMetricsInfo.BucketUsedNamespace, bucketInfo.getUsedNamespace()) .addGauge(BucketMetricsInfo.BucketAvailableBytes, availableSpace); } } @@ -103,6 +105,7 @@ enum BucketMetricsInfo implements MetricsInfo { BucketQuotaBytes("Bucket quota in bytes"), BucketSnapshotUsedBytes("Bucket quota bytes held in snapshots"), BucketQuotaNamespace("Bucket quota in namespace."), + BucketUsedNamespace("Bucket used namespace."), BucketAvailableBytes("Bucket available space."); private final String desc; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index b76e5aa52629..055481a40d4e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -991,6 +991,12 @@ public List listBuckets(final String volumeName, return bucketTable.cacheIterator(); } + @Override + public Iterator, CacheValue>> + getVolumeIterator() { + return volumeTable.cacheIterator(); + } + @Override public TableIterator> getKeyIterator() throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index db7717c81ea2..243cf86a3613 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -496,6 +496,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private final OzoneLockProvider ozoneLockProvider; private final OMPerformanceMetrics perfMetrics; private final BucketUtilizationMetrics bucketUtilizationMetrics; + private final VolumeUtilizationMetrics volumeUtilizationMetrics; private boolean fsSnapshotEnabled; @@ -761,6 +762,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) } bucketUtilizationMetrics = BucketUtilizationMetrics.create(metadataManager); + volumeUtilizationMetrics = VolumeUtilizationMetrics.create(metadataManager); omHostName = HddsUtils.getHostName(conf); } @@ -2485,6 +2487,10 @@ public boolean stop() { bucketUtilizationMetrics.unRegister(); } + if (volumeUtilizationMetrics != null) { + volumeUtilizationMetrics.unRegister(); + } + if (versionManager != null) { versionManager.close(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeUtilizationMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeUtilizationMetrics.java new file mode 100644 index 000000000000..92e6ed9e8724 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeUtilizationMetrics.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.hadoop.hdds.annotation.InterfaceAudience; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; + +/** + * A class for collecting and reporting volume utilization metrics. + *

+ * Available metrics: + *

    + *
  • Volume quota in bytes. + *
  • Volume used bytes (aggregated from all buckets in the volume). + *
  • Volume available bytes. Calculated from difference between volume quota and used bytes. + * If volume quota is not set then this metric shows -1 as value. + *
  • Volume quota in namespace (maximum number of buckets). + *
  • Volume used namespace (current number of buckets). + *
+ */ +@InterfaceAudience.Private +@Metrics(about = "Ozone Volume Utilization Metrics", context = OzoneConsts.OZONE) +public class VolumeUtilizationMetrics implements MetricsSource { + + private static final String SOURCE = VolumeUtilizationMetrics.class.getSimpleName(); + + private final OMMetadataManager metadataManager; + + public VolumeUtilizationMetrics(OMMetadataManager metadataManager) { + this.metadataManager = metadataManager; + } + + public static VolumeUtilizationMetrics create(OMMetadataManager metadataManager) { + MetricsSystem ms = DefaultMetricsSystem.instance(); + return ms.register(SOURCE, "Volume Utilization Metrics", new VolumeUtilizationMetrics(metadataManager)); + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + Map volumes = new HashMap<>(); + Iterator, CacheValue>> volumeIterator = metadataManager.getVolumeIterator(); + while (volumeIterator.hasNext()) { + Entry, CacheValue> entry = volumeIterator.next(); + OmVolumeArgs volumeArgs = entry.getValue().getCacheValue(); + if (volumeArgs != null) { + volumes.put(volumeArgs.getVolume(), volumeArgs); + } + } + + Map usedBytesPerVolume = new HashMap<>(); + Iterator, CacheValue>> bucketIterator = metadataManager.getBucketIterator(); + while (bucketIterator.hasNext()) { + Entry, CacheValue> entry = bucketIterator.next(); + OmBucketInfo bucketInfo = entry.getValue().getCacheValue(); + if (bucketInfo == null) { + continue; + } + usedBytesPerVolume.merge(bucketInfo.getVolumeName(), bucketInfo.getUsedBytes(), Long::sum); + } + + for (OmVolumeArgs volumeArgs : volumes.values()) { + long quotaInBytes = volumeArgs.getQuotaInBytes(); + long usedBytes = usedBytesPerVolume.getOrDefault(volumeArgs.getVolume(), 0L); + long availableBytes; + if (quotaInBytes == -1) { + availableBytes = -1; + } else { + availableBytes = Math.max(quotaInBytes - usedBytes, 0); + } + + collector.addRecord(SOURCE) + .setContext("Volume metrics") + .tag(VolumeMetricsInfo.VolumeName, volumeArgs.getVolume()) + .addGauge(VolumeMetricsInfo.VolumeQuotaBytes, quotaInBytes) + .addGauge(VolumeMetricsInfo.VolumeUsedBytes, usedBytes) + .addGauge(VolumeMetricsInfo.VolumeAvailableBytes, availableBytes) + .addGauge(VolumeMetricsInfo.VolumeQuotaNamespace, volumeArgs.getQuotaInNamespace()) + .addGauge(VolumeMetricsInfo.VolumeUsedNamespace, volumeArgs.getUsedNamespace()); + } + } + + public void unRegister() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + ms.unregisterSource(SOURCE); + } + + enum VolumeMetricsInfo implements MetricsInfo { + VolumeName("Volume name."), + VolumeQuotaBytes("Volume quota in bytes."), + VolumeUsedBytes("Bytes used across all buckets in the volume."), + VolumeAvailableBytes("Volume available space."), + VolumeQuotaNamespace("Volume quota in namespace."), + VolumeUsedNamespace("Volume used namespace."); + + private final String desc; + + VolumeMetricsInfo(String desc) { + this.desc = desc; + } + + @Override + public String description() { + return desc; + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketUtilizationMetrics.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketUtilizationMetrics.java index 653df6dd9201..97dc240ac451 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketUtilizationMetrics.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketUtilizationMetrics.java @@ -55,15 +55,17 @@ public class TestBucketUtilizationMetrics { private static final long QUOTA_IN_BYTES_2 = QUOTA_RESET; private static final long QUOTA_IN_NAMESPACE_1 = 1; private static final long QUOTA_IN_NAMESPACE_2 = 2; + private static final long USED_NAMESPACE_1 = 5; + private static final long USED_NAMESPACE_2 = 3; @Test void testBucketUtilizationMetrics() { OMMetadataManager omMetadataManager = mock(OMMetadataManager.class); Map.Entry, CacheValue> entry1 = createMockEntry(VOLUME_NAME_1, BUCKET_NAME_1, - USED_BYTES_1, SNAPSHOT_USED_BYTES_1, QUOTA_IN_BYTES_1, QUOTA_IN_NAMESPACE_1); + USED_BYTES_1, SNAPSHOT_USED_BYTES_1, QUOTA_IN_BYTES_1, QUOTA_IN_NAMESPACE_1, USED_NAMESPACE_1); Map.Entry, CacheValue> entry2 = createMockEntry(VOLUME_NAME_2, BUCKET_NAME_2, - USED_BYTES_2, SNAPSHOT_USED_BYTES_2, QUOTA_IN_BYTES_2, QUOTA_IN_NAMESPACE_2); + USED_BYTES_2, SNAPSHOT_USED_BYTES_2, QUOTA_IN_BYTES_2, QUOTA_IN_NAMESPACE_2, USED_NAMESPACE_2); Iterator, CacheValue>> bucketIterator = mock(Iterator.class); when(bucketIterator.hasNext()) @@ -96,6 +98,7 @@ void testBucketUtilizationMetrics() { verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketSnapshotUsedBytes, SNAPSHOT_USED_BYTES_1); verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketQuotaBytes, QUOTA_IN_BYTES_1); verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketQuotaNamespace, QUOTA_IN_NAMESPACE_1); + verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketUsedNamespace, USED_NAMESPACE_1); verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketAvailableBytes, QUOTA_IN_BYTES_1 - USED_BYTES_1 - SNAPSHOT_USED_BYTES_1); @@ -105,11 +108,13 @@ void testBucketUtilizationMetrics() { verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketSnapshotUsedBytes, SNAPSHOT_USED_BYTES_2); verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketQuotaBytes, QUOTA_IN_BYTES_2); verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketQuotaNamespace, QUOTA_IN_NAMESPACE_2); + verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketUsedNamespace, USED_NAMESPACE_2); verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketAvailableBytes, QUOTA_RESET); } private static Map.Entry, CacheValue> createMockEntry(String volumeName, - String bucketName, long usedBytes, long snapshotUsedBytes, long quotaInBytes, long quotaInNamespace) { + String bucketName, long usedBytes, long snapshotUsedBytes, long quotaInBytes, long quotaInNamespace, + long usedNamespace) { Map.Entry, CacheValue> entry = mock(Map.Entry.class); CacheValue cacheValue = mock(CacheValue.class); OmBucketInfo bucketInfo = mock(OmBucketInfo.class); @@ -120,6 +125,7 @@ private static Map.Entry, CacheValue> createMockE when(bucketInfo.getSnapshotUsedBytes()).thenReturn(snapshotUsedBytes); when(bucketInfo.getQuotaInBytes()).thenReturn(quotaInBytes); when(bucketInfo.getQuotaInNamespace()).thenReturn(quotaInNamespace); + when(bucketInfo.getUsedNamespace()).thenReturn(usedNamespace); when(bucketInfo.getTotalBucketSpace()).thenReturn(usedBytes + snapshotUsedBytes); when(cacheValue.getCacheValue()).thenReturn(bucketInfo); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVolumeUtilizationMetrics.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVolumeUtilizationMetrics.java new file mode 100644 index 000000000000..80c5933e622a --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVolumeUtilizationMetrics.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Iterator; +import java.util.Map; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.VolumeUtilizationMetrics.VolumeMetricsInfo; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.junit.jupiter.api.Test; + +/** + * Test class for VolumeUtilizationMetrics. + */ +public class TestVolumeUtilizationMetrics { + + private static final String VOLUME_NAME_1 = "volume1"; + private static final String VOLUME_NAME_2 = "volume2"; + private static final long QUOTA_IN_BYTES_1 = OzoneConsts.TB; + private static final long QUOTA_IN_BYTES_2 = -1; + private static final long QUOTA_IN_NAMESPACE_1 = 10; + private static final long QUOTA_IN_NAMESPACE_2 = -1; + private static final long USED_NAMESPACE_1 = 4; + private static final long USED_NAMESPACE_2 = 0; + private static final long BUCKET_USED_BYTES_1 = 100; + private static final long BUCKET_USED_BYTES_2 = 200; + + @Test + void testVolumeUtilizationMetrics() { + OMMetadataManager omMetadataManager = mock(OMMetadataManager.class); + + Map.Entry, CacheValue> entry1 = + createMockEntry(VOLUME_NAME_1, QUOTA_IN_BYTES_1, QUOTA_IN_NAMESPACE_1, USED_NAMESPACE_1); + Map.Entry, CacheValue> entry2 = + createMockEntry(VOLUME_NAME_2, QUOTA_IN_BYTES_2, QUOTA_IN_NAMESPACE_2, USED_NAMESPACE_2); + + Iterator, CacheValue>> volumeIterator = mock(Iterator.class); + when(volumeIterator.hasNext()) + .thenReturn(true) + .thenReturn(true) + .thenReturn(false); + + when(volumeIterator.next()) + .thenReturn(entry1) + .thenReturn(entry2); + + when(omMetadataManager.getVolumeIterator()).thenReturn(volumeIterator); + + Map.Entry, CacheValue> bucketEntry1 = + createMockBucketEntry(VOLUME_NAME_1, BUCKET_USED_BYTES_1); + Map.Entry, CacheValue> bucketEntry2 = + createMockBucketEntry(VOLUME_NAME_2, BUCKET_USED_BYTES_2); + + Iterator, CacheValue>> bucketIterator = mock(Iterator.class); + when(bucketIterator.hasNext()) + .thenReturn(true) + .thenReturn(true) + .thenReturn(false); + when(bucketIterator.next()) + .thenReturn(bucketEntry1) + .thenReturn(bucketEntry2); + + when(omMetadataManager.getBucketIterator()).thenReturn(bucketIterator); + + MetricsRecordBuilder mb = mock(MetricsRecordBuilder.class); + when(mb.setContext(anyString())).thenReturn(mb); + when(mb.tag(any(MetricsInfo.class), anyString())).thenReturn(mb); + when(mb.addGauge(any(MetricsInfo.class), anyInt())).thenReturn(mb); + when(mb.addGauge(any(MetricsInfo.class), anyLong())).thenReturn(mb); + + MetricsCollector metricsCollector = mock(MetricsCollector.class); + when(metricsCollector.addRecord(anyString())).thenReturn(mb); + + VolumeUtilizationMetrics volumeMetrics = new VolumeUtilizationMetrics(omMetadataManager); + volumeMetrics.getMetrics(metricsCollector, true); + + verify(mb, times(1)).tag(VolumeMetricsInfo.VolumeName, VOLUME_NAME_1); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaBytes, QUOTA_IN_BYTES_1); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedBytes, BUCKET_USED_BYTES_1); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeAvailableBytes, QUOTA_IN_BYTES_1 - BUCKET_USED_BYTES_1); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaNamespace, QUOTA_IN_NAMESPACE_1); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedNamespace, USED_NAMESPACE_1); + + verify(mb, times(1)).tag(VolumeMetricsInfo.VolumeName, VOLUME_NAME_2); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaBytes, QUOTA_IN_BYTES_2); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedBytes, BUCKET_USED_BYTES_2); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeAvailableBytes, QUOTA_IN_BYTES_2); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaNamespace, QUOTA_IN_NAMESPACE_2); + verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedNamespace, USED_NAMESPACE_2); + } + + private static Map.Entry, CacheValue> createMockBucketEntry( + String volumeName, long usedBytes) { + Map.Entry, CacheValue> entry = mock(Map.Entry.class); + CacheValue cacheValue = mock(CacheValue.class); + OmBucketInfo bucketInfo = mock(OmBucketInfo.class); + + when(bucketInfo.getVolumeName()).thenReturn(volumeName); + when(bucketInfo.getUsedBytes()).thenReturn(usedBytes); + + when(cacheValue.getCacheValue()).thenReturn(bucketInfo); + when(entry.getValue()).thenReturn(cacheValue); + + return entry; + } + + private static Map.Entry, CacheValue> createMockEntry( + String volumeName, long quotaInBytes, long quotaInNamespace, long usedNamespace) { + Map.Entry, CacheValue> entry = mock(Map.Entry.class); + CacheValue cacheValue = mock(CacheValue.class); + OmVolumeArgs volumeArgs = mock(OmVolumeArgs.class); + + when(volumeArgs.getVolume()).thenReturn(volumeName); + when(volumeArgs.getQuotaInBytes()).thenReturn(quotaInBytes); + when(volumeArgs.getQuotaInNamespace()).thenReturn(quotaInNamespace); + when(volumeArgs.getUsedNamespace()).thenReturn(usedNamespace); + + when(cacheValue.getCacheValue()).thenReturn(volumeArgs); + when(entry.getValue()).thenReturn(cacheValue); + + return entry; + } +} From 4e0c034bf9dc484132bb85fff87621edbe70e5a3 Mon Sep 17 00:00:00 2001 From: Prathmesh Sapate Date: Tue, 23 Jun 2026 16:29:29 +0530 Subject: [PATCH 2/2] HDDS-15333. Removed volume used bytes aggregation. --- .../ozone/om/VolumeUtilizationMetrics.java | 38 ++----------------- .../om/TestVolumeUtilizationMetrics.java | 38 ------------------- 2 files changed, 3 insertions(+), 73 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeUtilizationMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeUtilizationMetrics.java index 92e6ed9e8724..7ef2787fbe16 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeUtilizationMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeUtilizationMetrics.java @@ -17,9 +17,7 @@ package org.apache.hadoop.ozone.om; -import java.util.HashMap; import java.util.Iterator; -import java.util.Map; import java.util.Map.Entry; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -31,7 +29,6 @@ import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; /** @@ -40,9 +37,6 @@ * Available metrics: *
    *
  • Volume quota in bytes. - *
  • Volume used bytes (aggregated from all buckets in the volume). - *
  • Volume available bytes. Calculated from difference between volume quota and used bytes. - * If volume quota is not set then this metric shows -1 as value. *
  • Volume quota in namespace (maximum number of buckets). *
  • Volume used namespace (current number of buckets). *
@@ -66,43 +60,19 @@ public static VolumeUtilizationMetrics create(OMMetadataManager metadataManager) @Override public void getMetrics(MetricsCollector collector, boolean all) { - Map volumes = new HashMap<>(); Iterator, CacheValue>> volumeIterator = metadataManager.getVolumeIterator(); + while (volumeIterator.hasNext()) { Entry, CacheValue> entry = volumeIterator.next(); OmVolumeArgs volumeArgs = entry.getValue().getCacheValue(); - if (volumeArgs != null) { - volumes.put(volumeArgs.getVolume(), volumeArgs); - } - } - - Map usedBytesPerVolume = new HashMap<>(); - Iterator, CacheValue>> bucketIterator = metadataManager.getBucketIterator(); - while (bucketIterator.hasNext()) { - Entry, CacheValue> entry = bucketIterator.next(); - OmBucketInfo bucketInfo = entry.getValue().getCacheValue(); - if (bucketInfo == null) { + if (volumeArgs == null) { continue; } - usedBytesPerVolume.merge(bucketInfo.getVolumeName(), bucketInfo.getUsedBytes(), Long::sum); - } - - for (OmVolumeArgs volumeArgs : volumes.values()) { - long quotaInBytes = volumeArgs.getQuotaInBytes(); - long usedBytes = usedBytesPerVolume.getOrDefault(volumeArgs.getVolume(), 0L); - long availableBytes; - if (quotaInBytes == -1) { - availableBytes = -1; - } else { - availableBytes = Math.max(quotaInBytes - usedBytes, 0); - } collector.addRecord(SOURCE) .setContext("Volume metrics") .tag(VolumeMetricsInfo.VolumeName, volumeArgs.getVolume()) - .addGauge(VolumeMetricsInfo.VolumeQuotaBytes, quotaInBytes) - .addGauge(VolumeMetricsInfo.VolumeUsedBytes, usedBytes) - .addGauge(VolumeMetricsInfo.VolumeAvailableBytes, availableBytes) + .addGauge(VolumeMetricsInfo.VolumeQuotaBytes, volumeArgs.getQuotaInBytes()) .addGauge(VolumeMetricsInfo.VolumeQuotaNamespace, volumeArgs.getQuotaInNamespace()) .addGauge(VolumeMetricsInfo.VolumeUsedNamespace, volumeArgs.getUsedNamespace()); } @@ -116,8 +86,6 @@ public void unRegister() { enum VolumeMetricsInfo implements MetricsInfo { VolumeName("Volume name."), VolumeQuotaBytes("Volume quota in bytes."), - VolumeUsedBytes("Bytes used across all buckets in the volume."), - VolumeAvailableBytes("Volume available space."), VolumeQuotaNamespace("Volume quota in namespace."), VolumeUsedNamespace("Volume used namespace."); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVolumeUtilizationMetrics.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVolumeUtilizationMetrics.java index 80c5933e622a..0e10bb750b3c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVolumeUtilizationMetrics.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVolumeUtilizationMetrics.java @@ -35,7 +35,6 @@ import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.VolumeUtilizationMetrics.VolumeMetricsInfo; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.junit.jupiter.api.Test; @@ -52,8 +51,6 @@ public class TestVolumeUtilizationMetrics { private static final long QUOTA_IN_NAMESPACE_2 = -1; private static final long USED_NAMESPACE_1 = 4; private static final long USED_NAMESPACE_2 = 0; - private static final long BUCKET_USED_BYTES_1 = 100; - private static final long BUCKET_USED_BYTES_2 = 200; @Test void testVolumeUtilizationMetrics() { @@ -76,22 +73,6 @@ void testVolumeUtilizationMetrics() { when(omMetadataManager.getVolumeIterator()).thenReturn(volumeIterator); - Map.Entry, CacheValue> bucketEntry1 = - createMockBucketEntry(VOLUME_NAME_1, BUCKET_USED_BYTES_1); - Map.Entry, CacheValue> bucketEntry2 = - createMockBucketEntry(VOLUME_NAME_2, BUCKET_USED_BYTES_2); - - Iterator, CacheValue>> bucketIterator = mock(Iterator.class); - when(bucketIterator.hasNext()) - .thenReturn(true) - .thenReturn(true) - .thenReturn(false); - when(bucketIterator.next()) - .thenReturn(bucketEntry1) - .thenReturn(bucketEntry2); - - when(omMetadataManager.getBucketIterator()).thenReturn(bucketIterator); - MetricsRecordBuilder mb = mock(MetricsRecordBuilder.class); when(mb.setContext(anyString())).thenReturn(mb); when(mb.tag(any(MetricsInfo.class), anyString())).thenReturn(mb); @@ -106,34 +87,15 @@ void testVolumeUtilizationMetrics() { verify(mb, times(1)).tag(VolumeMetricsInfo.VolumeName, VOLUME_NAME_1); verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaBytes, QUOTA_IN_BYTES_1); - verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedBytes, BUCKET_USED_BYTES_1); - verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeAvailableBytes, QUOTA_IN_BYTES_1 - BUCKET_USED_BYTES_1); verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaNamespace, QUOTA_IN_NAMESPACE_1); verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedNamespace, USED_NAMESPACE_1); verify(mb, times(1)).tag(VolumeMetricsInfo.VolumeName, VOLUME_NAME_2); verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaBytes, QUOTA_IN_BYTES_2); - verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedBytes, BUCKET_USED_BYTES_2); - verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeAvailableBytes, QUOTA_IN_BYTES_2); verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaNamespace, QUOTA_IN_NAMESPACE_2); verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedNamespace, USED_NAMESPACE_2); } - private static Map.Entry, CacheValue> createMockBucketEntry( - String volumeName, long usedBytes) { - Map.Entry, CacheValue> entry = mock(Map.Entry.class); - CacheValue cacheValue = mock(CacheValue.class); - OmBucketInfo bucketInfo = mock(OmBucketInfo.class); - - when(bucketInfo.getVolumeName()).thenReturn(volumeName); - when(bucketInfo.getUsedBytes()).thenReturn(usedBytes); - - when(cacheValue.getCacheValue()).thenReturn(bucketInfo); - when(entry.getValue()).thenReturn(cacheValue); - - return entry; - } - private static Map.Entry, CacheValue> createMockEntry( String volumeName, long quotaInBytes, long quotaInNamespace, long usedNamespace) { Map.Entry, CacheValue> entry = mock(Map.Entry.class);