HDDS-15333. Expose volume and bucket utilization metrics on OM /prom

hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java

@@ -561,6 +561,9 @@ List<OmMultipartUpload> getMultipartUploadKeys(String volumeName,
   Iterator<Map.Entry<CacheKey<String>, CacheValue<OmBucketInfo>>>
       getBucketIterator();
 
+  Iterator<Map.Entry<CacheKey<String>, CacheValue<OmVolumeArgs>>>
+      getVolumeIterator();
+
   TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
       getKeyIterator() throws IOException;
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketUtilizationMetrics.java

@@ -37,10 +37,11 @@
  * Available metrics:
  * <ul>
  *   <li>Bytes used in bucket.
- *   <li>Bucket quote in bytes.
+ *   <li>Bucket quota in bytes.
  *   <li>Bucket quota in namespace.
+ *   <li>Bucket used namespace (number of keys/directories in bucket).
  *   <li>Bucket available space. Calculated from difference between used bytes in bucket and bucket quota.
- *   If bucket quote is not set then this metric show -1 as value.
+ *   If bucket quota is not set then this metric shows -1 as value.
  * </ul>
  */
 @InterfaceAudience.Private
@@ -87,6 +88,7 @@ public void getMetrics(MetricsCollector collector, boolean all) {
           .addGauge(BucketMetricsInfo.BucketSnapshotUsedBytes, bucketInfo.getSnapshotUsedBytes())
           .addGauge(BucketMetricsInfo.BucketQuotaBytes, bucketInfo.getQuotaInBytes())
           .addGauge(BucketMetricsInfo.BucketQuotaNamespace, bucketInfo.getQuotaInNamespace())
+          .addGauge(BucketMetricsInfo.BucketUsedNamespace, bucketInfo.getUsedNamespace())
           .addGauge(BucketMetricsInfo.BucketAvailableBytes, availableSpace);
     }
   }
@@ -103,6 +105,7 @@ enum BucketMetricsInfo implements MetricsInfo {
     BucketQuotaBytes("Bucket quota in bytes"),
     BucketSnapshotUsedBytes("Bucket quota bytes held in snapshots"),
     BucketQuotaNamespace("Bucket quota in namespace."),
+    BucketUsedNamespace("Bucket used namespace."),
     BucketAvailableBytes("Bucket available space.");
 
     private final String desc;

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -991,6 +991,12 @@ public List<OmBucketInfo> listBuckets(final String volumeName,
     return bucketTable.cacheIterator();
   }
 
+  @Override
+  public Iterator<Map.Entry<CacheKey<String>, CacheValue<OmVolumeArgs>>>
+      getVolumeIterator() {
+    return volumeTable.cacheIterator();
+  }
+
   @Override
   public TableIterator<String, ? extends KeyValue<String, OmKeyInfo>>
       getKeyIterator() throws IOException {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -496,6 +496,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   private final OzoneLockProvider ozoneLockProvider;
   private final OMPerformanceMetrics perfMetrics;
   private final BucketUtilizationMetrics bucketUtilizationMetrics;
+  private final VolumeUtilizationMetrics volumeUtilizationMetrics;
 
   private boolean fsSnapshotEnabled;
 
@@ -761,6 +762,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
     }
 
     bucketUtilizationMetrics = BucketUtilizationMetrics.create(metadataManager);
+    volumeUtilizationMetrics = VolumeUtilizationMetrics.create(metadataManager);
     omHostName = HddsUtils.getHostName(conf);
   }
 
@@ -2485,6 +2487,10 @@ public boolean stop() {
         bucketUtilizationMetrics.unRegister();
       }
 
+      if (volumeUtilizationMetrics != null) {
+        volumeUtilizationMetrics.unRegister();
+      }
+
       if (versionManager != null) {
         versionManager.close();
       }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeUtilizationMetrics.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+
+/**
+ * A class for collecting and reporting volume utilization metrics.
+ * <p>
+ * Available metrics:
+ * <ul>
+ *   <li>Volume quota in bytes.
+ *   <li>Volume quota in namespace (maximum number of buckets).
+ *   <li>Volume used namespace (current number of buckets).
+ * </ul>
+ */
+@InterfaceAudience.Private
+@Metrics(about = "Ozone Volume Utilization Metrics", context = OzoneConsts.OZONE)
+public class VolumeUtilizationMetrics implements MetricsSource {
+
+  private static final String SOURCE = VolumeUtilizationMetrics.class.getSimpleName();
+
+  private final OMMetadataManager metadataManager;
+
+  public VolumeUtilizationMetrics(OMMetadataManager metadataManager) {
+    this.metadataManager = metadataManager;
+  }
+
+  public static VolumeUtilizationMetrics create(OMMetadataManager metadataManager) {
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    return ms.register(SOURCE, "Volume Utilization Metrics", new VolumeUtilizationMetrics(metadataManager));
+  }
+
+  @Override
+  public void getMetrics(MetricsCollector collector, boolean all) {
+    Iterator<Entry<CacheKey<String>, CacheValue<OmVolumeArgs>>> volumeIterator = metadataManager.getVolumeIterator();
+
+    while (volumeIterator.hasNext()) {
+      Entry<CacheKey<String>, CacheValue<OmVolumeArgs>> entry = volumeIterator.next();
+      OmVolumeArgs volumeArgs = entry.getValue().getCacheValue();
+      if (volumeArgs == null) {
+        continue;
+      }
+
+      collector.addRecord(SOURCE)
+          .setContext("Volume metrics")
+          .tag(VolumeMetricsInfo.VolumeName, volumeArgs.getVolume())
+          .addGauge(VolumeMetricsInfo.VolumeQuotaBytes, volumeArgs.getQuotaInBytes())
+          .addGauge(VolumeMetricsInfo.VolumeQuotaNamespace, volumeArgs.getQuotaInNamespace())
+          .addGauge(VolumeMetricsInfo.VolumeUsedNamespace, volumeArgs.getUsedNamespace());
+    }
+  }
+
+  public void unRegister() {
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    ms.unregisterSource(SOURCE);
+  }
+
+  enum VolumeMetricsInfo implements MetricsInfo {
+    VolumeName("Volume name."),
+    VolumeQuotaBytes("Volume quota in bytes."),
+    VolumeQuotaNamespace("Volume quota in namespace."),
+    VolumeUsedNamespace("Volume used namespace.");
+
+    private final String desc;
+
+    VolumeMetricsInfo(String desc) {
+      this.desc = desc;
+    }
+
+    @Override
+    public String description() {
+      return desc;
+    }
+  }
+}

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketUtilizationMetrics.java

@@ -55,15 +55,17 @@ public class TestBucketUtilizationMetrics {
   private static final long QUOTA_IN_BYTES_2 = QUOTA_RESET;
   private static final long QUOTA_IN_NAMESPACE_1 = 1;
   private static final long QUOTA_IN_NAMESPACE_2 = 2;
+  private static final long USED_NAMESPACE_1 = 5;
+  private static final long USED_NAMESPACE_2 = 3;
 
   @Test
   void testBucketUtilizationMetrics() {
     OMMetadataManager omMetadataManager = mock(OMMetadataManager.class);
 
     Map.Entry<CacheKey<String>, CacheValue<OmBucketInfo>> entry1 = createMockEntry(VOLUME_NAME_1, BUCKET_NAME_1,
-        USED_BYTES_1, SNAPSHOT_USED_BYTES_1, QUOTA_IN_BYTES_1, QUOTA_IN_NAMESPACE_1);
+        USED_BYTES_1, SNAPSHOT_USED_BYTES_1, QUOTA_IN_BYTES_1, QUOTA_IN_NAMESPACE_1, USED_NAMESPACE_1);
     Map.Entry<CacheKey<String>, CacheValue<OmBucketInfo>> entry2 = createMockEntry(VOLUME_NAME_2, BUCKET_NAME_2,
-        USED_BYTES_2, SNAPSHOT_USED_BYTES_2, QUOTA_IN_BYTES_2, QUOTA_IN_NAMESPACE_2);
+        USED_BYTES_2, SNAPSHOT_USED_BYTES_2, QUOTA_IN_BYTES_2, QUOTA_IN_NAMESPACE_2, USED_NAMESPACE_2);
 
     Iterator<Map.Entry<CacheKey<String>, CacheValue<OmBucketInfo>>> bucketIterator = mock(Iterator.class);
     when(bucketIterator.hasNext())
@@ -96,6 +98,7 @@ void testBucketUtilizationMetrics() {
     verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketSnapshotUsedBytes, SNAPSHOT_USED_BYTES_1);
     verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketQuotaBytes, QUOTA_IN_BYTES_1);
     verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketQuotaNamespace, QUOTA_IN_NAMESPACE_1);
+    verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketUsedNamespace, USED_NAMESPACE_1);
     verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketAvailableBytes,
         QUOTA_IN_BYTES_1 - USED_BYTES_1 - SNAPSHOT_USED_BYTES_1);
 
@@ -105,11 +108,13 @@ void testBucketUtilizationMetrics() {
     verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketSnapshotUsedBytes, SNAPSHOT_USED_BYTES_2);
     verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketQuotaBytes, QUOTA_IN_BYTES_2);
     verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketQuotaNamespace, QUOTA_IN_NAMESPACE_2);
+    verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketUsedNamespace, USED_NAMESPACE_2);
     verify(mb, times(1)).addGauge(BucketMetricsInfo.BucketAvailableBytes, QUOTA_RESET);
   }
 
   private static Map.Entry<CacheKey<String>, CacheValue<OmBucketInfo>> createMockEntry(String volumeName,
-      String bucketName, long usedBytes, long snapshotUsedBytes, long quotaInBytes, long quotaInNamespace) {
+      String bucketName, long usedBytes, long snapshotUsedBytes, long quotaInBytes, long quotaInNamespace,
+      long usedNamespace) {
     Map.Entry<CacheKey<String>, CacheValue<OmBucketInfo>> entry = mock(Map.Entry.class);
     CacheValue<OmBucketInfo> cacheValue = mock(CacheValue.class);
     OmBucketInfo bucketInfo = mock(OmBucketInfo.class);
@@ -120,6 +125,7 @@ private static Map.Entry<CacheKey<String>, CacheValue<OmBucketInfo>> createMockE
     when(bucketInfo.getSnapshotUsedBytes()).thenReturn(snapshotUsedBytes);
     when(bucketInfo.getQuotaInBytes()).thenReturn(quotaInBytes);
     when(bucketInfo.getQuotaInNamespace()).thenReturn(quotaInNamespace);
+    when(bucketInfo.getUsedNamespace()).thenReturn(usedNamespace);
     when(bucketInfo.getTotalBucketSpace()).thenReturn(usedBytes + snapshotUsedBytes);
 
     when(cacheValue.getCacheValue()).thenReturn(bucketInfo);

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestVolumeUtilizationMetrics.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Iterator;
+import java.util.Map;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.VolumeUtilizationMetrics.VolumeMetricsInfo;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test class for VolumeUtilizationMetrics.
+ */
+public class TestVolumeUtilizationMetrics {
+
+  private static final String VOLUME_NAME_1 = "volume1";
+  private static final String VOLUME_NAME_2 = "volume2";
+  private static final long QUOTA_IN_BYTES_1 = OzoneConsts.TB;
+  private static final long QUOTA_IN_BYTES_2 = -1;
+  private static final long QUOTA_IN_NAMESPACE_1 = 10;
+  private static final long QUOTA_IN_NAMESPACE_2 = -1;
+  private static final long USED_NAMESPACE_1 = 4;
+  private static final long USED_NAMESPACE_2 = 0;
+
+  @Test
+  void testVolumeUtilizationMetrics() {
+    OMMetadataManager omMetadataManager = mock(OMMetadataManager.class);
+
+    Map.Entry<CacheKey<String>, CacheValue<OmVolumeArgs>> entry1 =
+        createMockEntry(VOLUME_NAME_1, QUOTA_IN_BYTES_1, QUOTA_IN_NAMESPACE_1, USED_NAMESPACE_1);
+    Map.Entry<CacheKey<String>, CacheValue<OmVolumeArgs>> entry2 =
+        createMockEntry(VOLUME_NAME_2, QUOTA_IN_BYTES_2, QUOTA_IN_NAMESPACE_2, USED_NAMESPACE_2);
+
+    Iterator<Map.Entry<CacheKey<String>, CacheValue<OmVolumeArgs>>> volumeIterator = mock(Iterator.class);
+    when(volumeIterator.hasNext())
+        .thenReturn(true)
+        .thenReturn(true)
+        .thenReturn(false);
+
+    when(volumeIterator.next())
+        .thenReturn(entry1)
+        .thenReturn(entry2);
+
+    when(omMetadataManager.getVolumeIterator()).thenReturn(volumeIterator);
+
+    MetricsRecordBuilder mb = mock(MetricsRecordBuilder.class);
+    when(mb.setContext(anyString())).thenReturn(mb);
+    when(mb.tag(any(MetricsInfo.class), anyString())).thenReturn(mb);
+    when(mb.addGauge(any(MetricsInfo.class), anyInt())).thenReturn(mb);
+    when(mb.addGauge(any(MetricsInfo.class), anyLong())).thenReturn(mb);
+
+    MetricsCollector metricsCollector = mock(MetricsCollector.class);
+    when(metricsCollector.addRecord(anyString())).thenReturn(mb);
+
+    VolumeUtilizationMetrics volumeMetrics = new VolumeUtilizationMetrics(omMetadataManager);
+    volumeMetrics.getMetrics(metricsCollector, true);
+
+    verify(mb, times(1)).tag(VolumeMetricsInfo.VolumeName, VOLUME_NAME_1);
+    verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaBytes, QUOTA_IN_BYTES_1);
+    verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaNamespace, QUOTA_IN_NAMESPACE_1);
+    verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedNamespace, USED_NAMESPACE_1);
+
+    verify(mb, times(1)).tag(VolumeMetricsInfo.VolumeName, VOLUME_NAME_2);
+    verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaBytes, QUOTA_IN_BYTES_2);
+    verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeQuotaNamespace, QUOTA_IN_NAMESPACE_2);
+    verify(mb, times(1)).addGauge(VolumeMetricsInfo.VolumeUsedNamespace, USED_NAMESPACE_2);
+  }
+
+  private static Map.Entry<CacheKey<String>, CacheValue<OmVolumeArgs>> createMockEntry(
+      String volumeName, long quotaInBytes, long quotaInNamespace, long usedNamespace) {
+    Map.Entry<CacheKey<String>, CacheValue<OmVolumeArgs>> entry = mock(Map.Entry.class);
+    CacheValue<OmVolumeArgs> cacheValue = mock(CacheValue.class);
+    OmVolumeArgs volumeArgs = mock(OmVolumeArgs.class);
+
+    when(volumeArgs.getVolume()).thenReturn(volumeName);
+    when(volumeArgs.getQuotaInBytes()).thenReturn(quotaInBytes);
+    when(volumeArgs.getQuotaInNamespace()).thenReturn(quotaInNamespace);
+    when(volumeArgs.getUsedNamespace()).thenReturn(usedNamespace);
+
+    when(cacheValue.getCacheValue()).thenReturn(volumeArgs);
+    when(entry.getValue()).thenReturn(cacheValue);
+
+    return entry;
+  }
+}